feat: add some utility scripts ✨ (#738)

Author: geniuspatrick

package.sh

@@ -0,0 +1,58 @@
+# Utility Scripts
+
+This folder is a collection of utility scripts, listed and explained below.
+
+> All scripts need to be run in the root path of project, unless otherwise noted.
+
+## gen_benchmark.py
+
+Generating benchmark by collecting results from [configs](../configs) folder. Usage:
+
+```shell
+python ./scripts/gen_benchmark.py
+```
+
+It will generate a markdown file, named as `benchmark_results.md`.
+
+## package.sh(Deprecated)
+
+Making wheel package of `mindcv` and sha256sum of the wheel files. Usage:
+
+```shell
+./scripts/package.sh
+```
+
+**New**! Just simply run the following command to make the wheel:
+
+```shell
+python -m build
+```
+
+## launch_dist.sh or launch_dist.py
+
+A simple clean launcher for distributed training on **_Ascend_**.
+Following [instruction](https://www.mindspore.cn/tutorials/experts/zh-CN/r2.1/parallel/startup_method.html) from Mindspore,
+except launching distributed training with `mpirun`, we can also use multiprocess
+with multi-card networking configuration `rank_table.json` to manually start a process on each card.
+To get `rank_table.json` on your machine, try the hccl tools from [here](https://gitee.com/mindspore/models/tree/master/utils/hccl_tools).
+
+> After you get the `rank_table.json`, replace the `"/path/to/rank_table.json"` in `launch_dist.sh` with the actual path.
+
+Now, you can replace your standalone launching with distributed launching:
+
+```diff
+- python script.py --arg1=value1 --arg2=value2
++ ./scripts/launch_dist.sh script.py --arg1=value1 --arg2=value2
+```
+
+where `--arg*` are arguments of `script.py`.
+
+For example:
+
+```shell
+./scripts/launch_dist.sh train.py --config=configs/resnet/resnet_50_ascend.yaml --data_dir=/my/awesome/dataset
+```
+
+> Note: Don't forget to check the argument `--distribute` if you are using `train.py` or `train_with_func.py`!
+
+For anyone who hates shell scripts, we offer python scripts `launch_dist.py` as well. Both are used in the same way!

scripts/README.md

@@ -0,0 +1,31 @@
+#!/usr/bin/env python
+# Usage:
+# ./scripts/launch_dist.py script.py --arg1=value1 --arg2=value2
+# Example:
+# ./scripts/launch_dist.py train.py --config=configs/resnet/resnet_50_ascend.yaml --data_dir=/my/awesome/dataset
+
+import multiprocessing as mp
+import os
+import sys
+
+BIAS = 0
+RANK_SIZE = 8
+RANK_TABLE_FILE = "/path/to/rank_table.json"
+
+
+def worker(rank_id, script, args):
+    os.environ["RANK_ID"] = f"{rank_id}"  # logical id
+    os.environ["DEVICE_ID"] = f"{rank_id + BIAS}"  # physical id
+    os.environ["RANK_TABLE_FILE"] = RANK_TABLE_FILE
+    print(f"Launching rank: {os.getenv('RANK_ID')}, device: {os.getenv('DEVICE_ID')}, pid: {os.getpid()}")
+    os.system(f"python -u {script} {args}")
+
+
+if __name__ == "__main__":
+    mp.set_start_method("spawn")
+
+    script_, args_ = sys.argv[1], " ".join(sys.argv[2:])
+    print(f"Script: {script_}, Args: {args_}")
+    processes = [mp.Process(target=worker, args=(i, script_, args_)) for i in range(RANK_SIZE)]
+    [p.start() for p in processes]
+    [p.join() for p in processes]

scripts/launch_dist.py

@@ -0,0 +1,22 @@
+#!/bin/bash
+# Usage:
+# ./scripts/launch_dist.sh script.py --arg1=value1 --arg2=value2
+# Example:
+# ./scripts/launch_dist.sh train.py --config=configs/resnet/resnet_50_ascend.yaml --data_dir=/my/awesome/dataset
+
+export RANK_SIZE=8
+export RANK_TABLE_FILE="/path/to/rank_table.json"
+
+
+echo "Script: $1, Args: ${@:2}"  # ${parameter:offset:length}
+
+# trap SIGINT to execute kill 0, which will kill all processes
+trap 'kill 0' SIGINT
+for ((i = 0; i < ${RANK_SIZE}; i++)); do
+    export RANK_ID=$i
+    export DEVICE_ID=$i
+    echo "Launching rank: ${RANK_ID}, device: ${DEVICE_ID}"
+    python -u $@ &
+done
+# wait for all processes to finish
+wait

scripts/launch_dist.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+set -e
+
+BASE_PATH=$(cd "$(dirname $0)/.."; pwd)
+OUTPUT_PATH="${BASE_PATH}/output"
+
+
+if [[ -d "${OUTPUT_PATH}" ]]; then
+    rm -rf "${OUTPUT_PATH}"
+fi
+mkdir -pv "${OUTPUT_PATH}"
+
+python ${BASE_PATH}/setup.py bdist_wheel
+
+mv ${BASE_PATH}/dist/*whl ${OUTPUT_PATH}
+
+cd "${OUTPUT_PATH}" || exit
+PACKAGE_LIST=$(ls mindcv-*.whl) || exit
+for PACKAGE_NAME in ${PACKAGE_LIST}; do
+    echo "writing sha256sum of ${PACKAGE_NAME}"
+    sha256sum -b "${PACKAGE_NAME}" > "${PACKAGE_NAME}.sha256"
+done