From 1407fa23ee606d270c88afab3075783754336c49 Mon Sep 17 00:00:00 2001 From: Marcel Wilnicki Date: Fri, 8 Mar 2024 11:32:29 +0100 Subject: [PATCH 01/35] first commit --- .../unet_3d/kits_19/run.py | 70 +++++++++---------- 1 file changed, 32 insertions(+), 38 deletions(-) diff --git a/computer_vision/semantic_segmentation/unet_3d/kits_19/run.py b/computer_vision/semantic_segmentation/unet_3d/kits_19/run.py index 4bb575266..c5e21ffa3 100644 --- a/computer_vision/semantic_segmentation/unet_3d/kits_19/run.py +++ b/computer_vision/semantic_segmentation/unet_3d/kits_19/run.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # Copyright (c) 2022, Ampere Computing LLC -import argparse +# import argparse import numpy as np import tensorflow as tf @@ -9,34 +9,25 @@ from utils.cv.kits import KiTS19 from utils.benchmark import run_model -from utils.misc import print_goodbye_message_and_die -def parse_args(): - parser = argparse.ArgumentParser(description="Run 3D Unet KiTS 2019 model.") - parser.add_argument("-m", "--model_path", - type=str, - help="path to the model") - parser.add_argument("-p", "--precision", - type=str, choices=["fp32"], required=True, - help="precision of the model provided") - parser.add_argument("-f", "--framework", - type=str, default="tf", - choices=["tf"], - help="specify the framework in which a model should be run") - parser.add_argument("--timeout", - type=float, default=60.0, - help="timeout in seconds") - parser.add_argument("--num_runs", - type=int, - help="number of passes through network to execute") - parser.add_argument("--kits_path", - type=str, - help="path to directory with KiTS19 dataset") - return parser.parse_args() +def run_tf_fp(model_path, num_runs, timeout, kits_path): + from utils.tf import TFSavedModelRunner + def run_single_pass(tf_runner, kits): + output = tf_runner.run(1, tf.constant(np.expand_dims(kits.get_input_array(), axis=0))) + output = output["output_0"] + kits.submit_predictions(output) -def run_tf_fp(model_path, num_runs, timeout, kits_path): + dataset = KiTS19(dataset_dir_path=kits_path) + runner = TFSavedModelRunner() + saved_model_loaded = tf.saved_model.load(model_path, tags=[tag_constants.SERVING]) + runner.model = saved_model_loaded.signatures['serving_default'] + + return run_model(run_single_pass, runner, dataset, 1, num_runs, timeout) + + +def run_pytorch_fp(model_path, num_runs, timeout, kits_path): from utils.tf import TFSavedModelRunner def run_single_pass(tf_runner, kits): @@ -56,22 +47,25 @@ def run_tf_fp32(model_path, num_runs, timeout, kits_path, **kwargs): return run_tf_fp(model_path, num_runs, timeout, kits_path) +def run_pytorch_fp32(model_path, num_runs, timeout, kits_path, **kwargs): + return run_tf_fp(model_path, num_runs, timeout, kits_path) + + def main(): - args = parse_args() - if args.framework == "tf": - if args.model_path is None: - print_goodbye_message_and_die( - "a path to model is unspecified!") - - if args.precision == "fp32": - run_tf_fp32(**vars(args)) - else: - print_goodbye_message_and_die( - "this model seems to be unsupported in a specified precision: " + args.precision) + from utils.helpers import DefaultArgParser + parser = DefaultArgParser(["tf", "pytorch"]) + parser.require_model_path() + parser.add_argument("--kits_path", + type=str, + help="path to directory with KiTS19 dataset") + args = parser.parse() + print(args) + quit() + if args.framework == 'tf': + run_tf_fp32(**vars(parser.parse())) else: - print_goodbye_message_and_die( - "this model seems to be unsupported in a specified framework: " + args.framework) + run_pytorch_fp32(**vars(parser.parse())) if __name__ == "__main__": From 7683f0ceab6ffb2f5ce2629d586c445f7f1e1f18 Mon Sep 17 00:00:00 2001 From: Marcel Wilnicki Date: Fri, 8 Mar 2024 13:46:00 +0100 Subject: [PATCH 02/35] wip --- computer_vision/semantic_segmentation/unet_3d/kits_19/run.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/computer_vision/semantic_segmentation/unet_3d/kits_19/run.py b/computer_vision/semantic_segmentation/unet_3d/kits_19/run.py index c5e21ffa3..2da4e329c 100644 --- a/computer_vision/semantic_segmentation/unet_3d/kits_19/run.py +++ b/computer_vision/semantic_segmentation/unet_3d/kits_19/run.py @@ -60,8 +60,6 @@ def main(): help="path to directory with KiTS19 dataset") args = parser.parse() - print(args) - quit() if args.framework == 'tf': run_tf_fp32(**vars(parser.parse())) else: From 0dda31cdb504efe9bcbbcfde2521715ea2828156 Mon Sep 17 00:00:00 2001 From: Marcel Wilnicki Date: Fri, 8 Mar 2024 15:49:06 +0100 Subject: [PATCH 03/35] wip --- .../semantic_segmentation/unet_3d/kits_19/run.py | 14 ++++++-------- .../text_generation/llama2/run.py | 2 ++ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/computer_vision/semantic_segmentation/unet_3d/kits_19/run.py b/computer_vision/semantic_segmentation/unet_3d/kits_19/run.py index 2da4e329c..42f555296 100644 --- a/computer_vision/semantic_segmentation/unet_3d/kits_19/run.py +++ b/computer_vision/semantic_segmentation/unet_3d/kits_19/run.py @@ -1,8 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # Copyright (c) 2022, Ampere Computing LLC -# import argparse - +import torch import numpy as np import tensorflow as tf from tensorflow.python.saved_model import tag_constants @@ -28,17 +27,16 @@ def run_single_pass(tf_runner, kits): def run_pytorch_fp(model_path, num_runs, timeout, kits_path): - from utils.tf import TFSavedModelRunner + from utils.pytorch import PyTorchRunnerV2 - def run_single_pass(tf_runner, kits): - output = tf_runner.run(1, tf.constant(np.expand_dims(kits.get_input_array(), axis=0))) + def run_single_pass(pytorch_runner, kits): + output = pytorch_runner.run(1, tf.constant(np.expand_dims(kits.get_input_array(), axis=0))) output = output["output_0"] kits.submit_predictions(output) dataset = KiTS19(dataset_dir_path=kits_path) - runner = TFSavedModelRunner() - saved_model_loaded = tf.saved_model.load(model_path, tags=[tag_constants.SERVING]) - runner.model = saved_model_loaded.signatures['serving_default'] + model = torch.jit.load(model_path) + runner = PyTorchRunnerV2(model) return run_model(run_single_pass, runner, dataset, 1, num_runs, timeout) diff --git a/natural_language_processing/text_generation/llama2/run.py b/natural_language_processing/text_generation/llama2/run.py index 9a65a64b7..ced83458c 100644 --- a/natural_language_processing/text_generation/llama2/run.py +++ b/natural_language_processing/text_generation/llama2/run.py @@ -39,9 +39,11 @@ def run_single_pass(pytorch_runner, _dataset): def run_pytorch_fp32(model_name, batch_size, num_runs, timeout, dataset_path, **kwargs): return run_pytorch(model_name, batch_size, num_runs, timeout, dataset_path) + def run_pytorch_fp16(model_name, batch_size, num_runs, timeout, dataset_path, **kwargs): return run_pytorch(model_name, batch_size, num_runs, timeout, dataset_path, use_torch_fp16=True) + def main(): from utils.helpers import DefaultArgParser llama_variants = ["meta-llama/Llama-2-7b-chat-hf", "meta-llama/Llama-2-13b-chat-hf"] From 1605b95ca1040752db4b5ffb2a57289d517b36ca Mon Sep 17 00:00:00 2001 From: Marcel Wilnicki Date: Fri, 8 Mar 2024 15:51:16 +0100 Subject: [PATCH 04/35] wip --- computer_vision/semantic_segmentation/unet_3d/kits_19/run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/computer_vision/semantic_segmentation/unet_3d/kits_19/run.py b/computer_vision/semantic_segmentation/unet_3d/kits_19/run.py index 42f555296..71143117a 100644 --- a/computer_vision/semantic_segmentation/unet_3d/kits_19/run.py +++ b/computer_vision/semantic_segmentation/unet_3d/kits_19/run.py @@ -46,7 +46,7 @@ def run_tf_fp32(model_path, num_runs, timeout, kits_path, **kwargs): def run_pytorch_fp32(model_path, num_runs, timeout, kits_path, **kwargs): - return run_tf_fp(model_path, num_runs, timeout, kits_path) + return run_pytorch_fp(model_path, num_runs, timeout, kits_path) def main(): From 8920897d3de3055a0cfc37e62267efe0ee8b92fd Mon Sep 17 00:00:00 2001 From: Marcel Wilnicki Date: Fri, 8 Mar 2024 16:02:53 +0100 Subject: [PATCH 05/35] wip --- computer_vision/semantic_segmentation/unet_3d/kits_19/run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/computer_vision/semantic_segmentation/unet_3d/kits_19/run.py b/computer_vision/semantic_segmentation/unet_3d/kits_19/run.py index 71143117a..66a9c1d82 100644 --- a/computer_vision/semantic_segmentation/unet_3d/kits_19/run.py +++ b/computer_vision/semantic_segmentation/unet_3d/kits_19/run.py @@ -35,7 +35,7 @@ def run_single_pass(pytorch_runner, kits): kits.submit_predictions(output) dataset = KiTS19(dataset_dir_path=kits_path) - model = torch.jit.load(model_path) + model = torch.jit.load(model_path, map_location=torch.device('cpu')) runner = PyTorchRunnerV2(model) return run_model(run_single_pass, runner, dataset, 1, num_runs, timeout) From 98900718eadf12acb00a1f0f2c729d461eb63b6e Mon Sep 17 00:00:00 2001 From: Marcel Wilnicki Date: Tue, 12 Mar 2024 15:49:49 +0100 Subject: [PATCH 06/35] wip --- computer_vision/semantic_segmentation/unet_3d/kits_19/run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/computer_vision/semantic_segmentation/unet_3d/kits_19/run.py b/computer_vision/semantic_segmentation/unet_3d/kits_19/run.py index 66a9c1d82..b0d0323db 100644 --- a/computer_vision/semantic_segmentation/unet_3d/kits_19/run.py +++ b/computer_vision/semantic_segmentation/unet_3d/kits_19/run.py @@ -30,7 +30,7 @@ def run_pytorch_fp(model_path, num_runs, timeout, kits_path): from utils.pytorch import PyTorchRunnerV2 def run_single_pass(pytorch_runner, kits): - output = pytorch_runner.run(1, tf.constant(np.expand_dims(kits.get_input_array(), axis=0))) + output = pytorch_runner.run(1, kits.get_input_array()) output = output["output_0"] kits.submit_predictions(output) From fbc378c6d7a35f7f0d306bafd3f38b3be48c6e39 Mon Sep 17 00:00:00 2001 From: Marcel Wilnicki Date: Tue, 12 Mar 2024 16:06:25 +0100 Subject: [PATCH 07/35] wip --- computer_vision/semantic_segmentation/unet_3d/kits_19/run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/computer_vision/semantic_segmentation/unet_3d/kits_19/run.py b/computer_vision/semantic_segmentation/unet_3d/kits_19/run.py index b0d0323db..47ad30360 100644 --- a/computer_vision/semantic_segmentation/unet_3d/kits_19/run.py +++ b/computer_vision/semantic_segmentation/unet_3d/kits_19/run.py @@ -30,7 +30,7 @@ def run_pytorch_fp(model_path, num_runs, timeout, kits_path): from utils.pytorch import PyTorchRunnerV2 def run_single_pass(pytorch_runner, kits): - output = pytorch_runner.run(1, kits.get_input_array()) + output = pytorch_runner.run(1, torch.from_numpy(np.expand_dims(kits.get_input_array(), axis=0))) output = output["output_0"] kits.submit_predictions(output) From 085efb546909435fb28d3efaa6e947f50640ef7f Mon Sep 17 00:00:00 2001 From: Marcel Wilnicki Date: Tue, 12 Mar 2024 19:56:33 +0100 Subject: [PATCH 08/35] wip --- computer_vision/semantic_segmentation/unet_3d/kits_19/run.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/computer_vision/semantic_segmentation/unet_3d/kits_19/run.py b/computer_vision/semantic_segmentation/unet_3d/kits_19/run.py index 47ad30360..1cb12e03c 100644 --- a/computer_vision/semantic_segmentation/unet_3d/kits_19/run.py +++ b/computer_vision/semantic_segmentation/unet_3d/kits_19/run.py @@ -31,8 +31,7 @@ def run_pytorch_fp(model_path, num_runs, timeout, kits_path): def run_single_pass(pytorch_runner, kits): output = pytorch_runner.run(1, torch.from_numpy(np.expand_dims(kits.get_input_array(), axis=0))) - output = output["output_0"] - kits.submit_predictions(output) + kits.submit_predictions(tf.convert_to_tensor(output.numpy())) dataset = KiTS19(dataset_dir_path=kits_path) model = torch.jit.load(model_path, map_location=torch.device('cpu')) From ea6d050e0054d912e1f4bb862fe24d425db69507 Mon Sep 17 00:00:00 2001 From: Marcel Wilnicki Date: Mon, 18 Mar 2024 15:52:48 +0100 Subject: [PATCH 09/35] wip --- .../unet_3d/kits_19/run.py | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/computer_vision/semantic_segmentation/unet_3d/kits_19/run.py b/computer_vision/semantic_segmentation/unet_3d/kits_19/run.py index 1cb12e03c..90ca84bde 100644 --- a/computer_vision/semantic_segmentation/unet_3d/kits_19/run.py +++ b/computer_vision/semantic_segmentation/unet_3d/kits_19/run.py @@ -40,6 +40,22 @@ def run_single_pass(pytorch_runner, kits): return run_model(run_single_pass, runner, dataset, 1, num_runs, timeout) +def run_pytorch_fp_1(model_path, num_runs, timeout, kits_path): + from utils.pytorch import PyTorchRunnerV2, apply_jit_script, apply_jit_trace + + def run_single_pass(pytorch_runner, kits): + output = pytorch_runner.run(1, torch.from_numpy(np.expand_dims(kits.get_input_array(), axis=0))) + kits.submit_predictions(tf.convert_to_tensor(output.numpy())) + + dataset = KiTS19(dataset_dir_path=kits_path) + model = torch.load(model_path) + model.eval() + model = apply_jit_script(model) + runner = PyTorchRunnerV2(model) + + return run_model(run_single_pass, runner, dataset, 1, num_runs, timeout) + + def run_tf_fp32(model_path, num_runs, timeout, kits_path, **kwargs): return run_tf_fp(model_path, num_runs, timeout, kits_path) @@ -48,6 +64,10 @@ def run_pytorch_fp32(model_path, num_runs, timeout, kits_path, **kwargs): return run_pytorch_fp(model_path, num_runs, timeout, kits_path) +def run_pytorch_fp32_1(model_path, num_runs, timeout, kits_path, **kwargs): + return run_pytorch_fp_1(model_path, num_runs, timeout, kits_path) + + def main(): from utils.helpers import DefaultArgParser parser = DefaultArgParser(["tf", "pytorch"]) From ae11b27658f96004fa77152830643cae55cbf2af Mon Sep 17 00:00:00 2001 From: Marcel Wilnicki Date: Mon, 18 Mar 2024 15:54:22 +0100 Subject: [PATCH 10/35] wip --- .../semantic_segmentation/unet_3d/kits_19/run.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/computer_vision/semantic_segmentation/unet_3d/kits_19/run.py b/computer_vision/semantic_segmentation/unet_3d/kits_19/run.py index 90ca84bde..a21ba18de 100644 --- a/computer_vision/semantic_segmentation/unet_3d/kits_19/run.py +++ b/computer_vision/semantic_segmentation/unet_3d/kits_19/run.py @@ -70,7 +70,7 @@ def run_pytorch_fp32_1(model_path, num_runs, timeout, kits_path, **kwargs): def main(): from utils.helpers import DefaultArgParser - parser = DefaultArgParser(["tf", "pytorch"]) + parser = DefaultArgParser(["tf", "pytorch", "pytorch_1"]) parser.require_model_path() parser.add_argument("--kits_path", type=str, @@ -79,8 +79,10 @@ def main(): args = parser.parse() if args.framework == 'tf': run_tf_fp32(**vars(parser.parse())) - else: + elif args.framework == 'pytorch': run_pytorch_fp32(**vars(parser.parse())) + else: + run_pytorch_fp32_1(**vars(parser.parse())) if __name__ == "__main__": From 188280bf887e6ada8f91b7cc601838e962a5d4a1 Mon Sep 17 00:00:00 2001 From: Marcel Wilnicki Date: Mon, 18 Mar 2024 15:59:05 +0100 Subject: [PATCH 11/35] wip --- .../unet_3d/kits_19/run.py | 38 ++++++++----------- 1 file changed, 16 insertions(+), 22 deletions(-) diff --git a/computer_vision/semantic_segmentation/unet_3d/kits_19/run.py b/computer_vision/semantic_segmentation/unet_3d/kits_19/run.py index a21ba18de..e5b45c33f 100644 --- a/computer_vision/semantic_segmentation/unet_3d/kits_19/run.py +++ b/computer_vision/semantic_segmentation/unet_3d/kits_19/run.py @@ -26,22 +26,22 @@ def run_single_pass(tf_runner, kits): return run_model(run_single_pass, runner, dataset, 1, num_runs, timeout) -def run_pytorch_fp(model_path, num_runs, timeout, kits_path): - from utils.pytorch import PyTorchRunnerV2 - - def run_single_pass(pytorch_runner, kits): - output = pytorch_runner.run(1, torch.from_numpy(np.expand_dims(kits.get_input_array(), axis=0))) - kits.submit_predictions(tf.convert_to_tensor(output.numpy())) +# def run_pytorch_fp(model_path, num_runs, timeout, kits_path): +# from utils.pytorch import PyTorchRunnerV2 +# +# def run_single_pass(pytorch_runner, kits): +# output = pytorch_runner.run(1, torch.from_numpy(np.expand_dims(kits.get_input_array(), axis=0))) +# kits.submit_predictions(tf.convert_to_tensor(output.numpy())) +# +# dataset = KiTS19(dataset_dir_path=kits_path) +# model = torch.jit.load(model_path, map_location=torch.device('cpu')) +# runner = PyTorchRunnerV2(model) +# +# return run_model(run_single_pass, runner, dataset, 1, num_runs, timeout) - dataset = KiTS19(dataset_dir_path=kits_path) - model = torch.jit.load(model_path, map_location=torch.device('cpu')) - runner = PyTorchRunnerV2(model) - return run_model(run_single_pass, runner, dataset, 1, num_runs, timeout) - - -def run_pytorch_fp_1(model_path, num_runs, timeout, kits_path): - from utils.pytorch import PyTorchRunnerV2, apply_jit_script, apply_jit_trace +def run_pytorch_fp(model_path, num_runs, timeout, kits_path): + from utils.pytorch import PyTorchRunnerV2, apply_jit_script def run_single_pass(pytorch_runner, kits): output = pytorch_runner.run(1, torch.from_numpy(np.expand_dims(kits.get_input_array(), axis=0))) @@ -64,13 +64,9 @@ def run_pytorch_fp32(model_path, num_runs, timeout, kits_path, **kwargs): return run_pytorch_fp(model_path, num_runs, timeout, kits_path) -def run_pytorch_fp32_1(model_path, num_runs, timeout, kits_path, **kwargs): - return run_pytorch_fp_1(model_path, num_runs, timeout, kits_path) - - def main(): from utils.helpers import DefaultArgParser - parser = DefaultArgParser(["tf", "pytorch", "pytorch_1"]) + parser = DefaultArgParser(["tf", "pytorch"]) parser.require_model_path() parser.add_argument("--kits_path", type=str, @@ -79,10 +75,8 @@ def main(): args = parser.parse() if args.framework == 'tf': run_tf_fp32(**vars(parser.parse())) - elif args.framework == 'pytorch': - run_pytorch_fp32(**vars(parser.parse())) else: - run_pytorch_fp32_1(**vars(parser.parse())) + run_pytorch_fp32(**vars(parser.parse())) if __name__ == "__main__": From 2d57f20b0b5183f758cf15fb004a865708c36763 Mon Sep 17 00:00:00 2001 From: Marcel Wilnicki Date: Mon, 18 Mar 2024 16:01:50 +0100 Subject: [PATCH 12/35] wip --- computer_vision/semantic_segmentation/unet_3d/kits_19/run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/computer_vision/semantic_segmentation/unet_3d/kits_19/run.py b/computer_vision/semantic_segmentation/unet_3d/kits_19/run.py index e5b45c33f..42a1cbd10 100644 --- a/computer_vision/semantic_segmentation/unet_3d/kits_19/run.py +++ b/computer_vision/semantic_segmentation/unet_3d/kits_19/run.py @@ -48,7 +48,7 @@ def run_single_pass(pytorch_runner, kits): kits.submit_predictions(tf.convert_to_tensor(output.numpy())) dataset = KiTS19(dataset_dir_path=kits_path) - model = torch.load(model_path) + model = torch.load(model_path, map_location=torch.device('cpu')) model.eval() model = apply_jit_script(model) runner = PyTorchRunnerV2(model) From 33cded26ff1a78b2309a9c09b07e6d0a1343af14 Mon Sep 17 00:00:00 2001 From: Marcel Wilnicki Date: Tue, 19 Mar 2024 10:36:42 +0100 Subject: [PATCH 13/35] wip --- .../unet_3d/kits_19/run.py | 36 ++++++++++--------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/computer_vision/semantic_segmentation/unet_3d/kits_19/run.py b/computer_vision/semantic_segmentation/unet_3d/kits_19/run.py index 42a1cbd10..fb777b6cb 100644 --- a/computer_vision/semantic_segmentation/unet_3d/kits_19/run.py +++ b/computer_vision/semantic_segmentation/unet_3d/kits_19/run.py @@ -26,36 +26,38 @@ def run_single_pass(tf_runner, kits): return run_model(run_single_pass, runner, dataset, 1, num_runs, timeout) -# def run_pytorch_fp(model_path, num_runs, timeout, kits_path): -# from utils.pytorch import PyTorchRunnerV2 -# -# def run_single_pass(pytorch_runner, kits): -# output = pytorch_runner.run(1, torch.from_numpy(np.expand_dims(kits.get_input_array(), axis=0))) -# kits.submit_predictions(tf.convert_to_tensor(output.numpy())) -# -# dataset = KiTS19(dataset_dir_path=kits_path) -# model = torch.jit.load(model_path, map_location=torch.device('cpu')) -# runner = PyTorchRunnerV2(model) -# -# return run_model(run_single_pass, runner, dataset, 1, num_runs, timeout) - - def run_pytorch_fp(model_path, num_runs, timeout, kits_path): - from utils.pytorch import PyTorchRunnerV2, apply_jit_script + from utils.pytorch import PyTorchRunnerV2 def run_single_pass(pytorch_runner, kits): output = pytorch_runner.run(1, torch.from_numpy(np.expand_dims(kits.get_input_array(), axis=0))) kits.submit_predictions(tf.convert_to_tensor(output.numpy())) dataset = KiTS19(dataset_dir_path=kits_path) - model = torch.load(model_path, map_location=torch.device('cpu')) + model = torch.jit.load(model_path, map_location=torch.device('cpu')) model.eval() - model = apply_jit_script(model) runner = PyTorchRunnerV2(model) return run_model(run_single_pass, runner, dataset, 1, num_runs, timeout) +# def run_pytorch_fp(model_path, num_runs, timeout, kits_path): +# from utils.pytorch import PyTorchRunnerV2, apply_jit_script +# +# def run_single_pass(pytorch_runner, kits): +# output = pytorch_runner.run(1, torch.from_numpy(np.expand_dims(kits.get_input_array(), axis=0))) +# kits.submit_predictions(tf.convert_to_tensor(output.numpy())) +# +# dataset = KiTS19(dataset_dir_path=kits_path) +# dlrm.load_state_dict(torch.load(model_path)["state_dict"]) +# model = torch.load(model_path, map_location=torch.device('cpu')) +# model.eval() +# model = apply_jit_script(model) +# runner = PyTorchRunnerV2(model) +# +# return run_model(run_single_pass, runner, dataset, 1, num_runs, timeout) + + def run_tf_fp32(model_path, num_runs, timeout, kits_path, **kwargs): return run_tf_fp(model_path, num_runs, timeout, kits_path) From ed80ebda9a39e11e7ff87826dd89e030b83ae7ac Mon Sep 17 00:00:00 2001 From: Marcel Wilnicki Date: Tue, 19 Mar 2024 11:36:57 +0100 Subject: [PATCH 14/35] wip --- computer_vision/semantic_segmentation/unet_3d/kits_19/run.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/computer_vision/semantic_segmentation/unet_3d/kits_19/run.py b/computer_vision/semantic_segmentation/unet_3d/kits_19/run.py index fb777b6cb..d51c6277d 100644 --- a/computer_vision/semantic_segmentation/unet_3d/kits_19/run.py +++ b/computer_vision/semantic_segmentation/unet_3d/kits_19/run.py @@ -27,7 +27,7 @@ def run_single_pass(tf_runner, kits): def run_pytorch_fp(model_path, num_runs, timeout, kits_path): - from utils.pytorch import PyTorchRunnerV2 + from utils.pytorch import PyTorchRunnerV2, apply_jit_script, apply_jit_trace def run_single_pass(pytorch_runner, kits): output = pytorch_runner.run(1, torch.from_numpy(np.expand_dims(kits.get_input_array(), axis=0))) @@ -36,6 +36,7 @@ def run_single_pass(pytorch_runner, kits): dataset = KiTS19(dataset_dir_path=kits_path) model = torch.jit.load(model_path, map_location=torch.device('cpu')) model.eval() + model = apply_jit_script(model) runner = PyTorchRunnerV2(model) return run_model(run_single_pass, runner, dataset, 1, num_runs, timeout) From f612b9500f1d8397d4566a2766ed9620ddee2352 Mon Sep 17 00:00:00 2001 From: Marcel Wilnicki Date: Tue, 19 Mar 2024 11:48:08 +0100 Subject: [PATCH 15/35] wip --- computer_vision/semantic_segmentation/unet_3d/kits_19/run.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/computer_vision/semantic_segmentation/unet_3d/kits_19/run.py b/computer_vision/semantic_segmentation/unet_3d/kits_19/run.py index d51c6277d..b966986fa 100644 --- a/computer_vision/semantic_segmentation/unet_3d/kits_19/run.py +++ b/computer_vision/semantic_segmentation/unet_3d/kits_19/run.py @@ -36,7 +36,8 @@ def run_single_pass(pytorch_runner, kits): dataset = KiTS19(dataset_dir_path=kits_path) model = torch.jit.load(model_path, map_location=torch.device('cpu')) model.eval() - model = apply_jit_script(model) + # model = apply_jit_script(model) + model = apply_jit_trace(model, torch.from_numpy(np.expand_dims(dataset.get_input_array(), axis=0))) runner = PyTorchRunnerV2(model) return run_model(run_single_pass, runner, dataset, 1, num_runs, timeout) From 2d514971d7cbd0a4afb3e3a02df825f09e6a5d38 Mon Sep 17 00:00:00 2001 From: Marcel Wilnicki Date: Tue, 19 Mar 2024 12:00:22 +0100 Subject: [PATCH 16/35] wip --- computer_vision/semantic_segmentation/unet_3d/kits_19/run.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/computer_vision/semantic_segmentation/unet_3d/kits_19/run.py b/computer_vision/semantic_segmentation/unet_3d/kits_19/run.py index b966986fa..8e12137b7 100644 --- a/computer_vision/semantic_segmentation/unet_3d/kits_19/run.py +++ b/computer_vision/semantic_segmentation/unet_3d/kits_19/run.py @@ -36,8 +36,9 @@ def run_single_pass(pytorch_runner, kits): dataset = KiTS19(dataset_dir_path=kits_path) model = torch.jit.load(model_path, map_location=torch.device('cpu')) model.eval() + torch.jit.freeze(model) # model = apply_jit_script(model) - model = apply_jit_trace(model, torch.from_numpy(np.expand_dims(dataset.get_input_array(), axis=0))) + # model = apply_jit_trace(model, torch.from_numpy(np.expand_dims(dataset.get_input_array(), axis=0))) runner = PyTorchRunnerV2(model) return run_model(run_single_pass, runner, dataset, 1, num_runs, timeout) From 35cd3b16e5ff11207779c9de87f2912ff673bcf3 Mon Sep 17 00:00:00 2001 From: Marcel Wilnicki Date: Fri, 22 Mar 2024 09:39:30 +0100 Subject: [PATCH 17/35] wip --- computer_vision/semantic_segmentation/unet_3d/kits_19/run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/computer_vision/semantic_segmentation/unet_3d/kits_19/run.py b/computer_vision/semantic_segmentation/unet_3d/kits_19/run.py index 8e12137b7..05e6574b7 100644 --- a/computer_vision/semantic_segmentation/unet_3d/kits_19/run.py +++ b/computer_vision/semantic_segmentation/unet_3d/kits_19/run.py @@ -36,7 +36,7 @@ def run_single_pass(pytorch_runner, kits): dataset = KiTS19(dataset_dir_path=kits_path) model = torch.jit.load(model_path, map_location=torch.device('cpu')) model.eval() - torch.jit.freeze(model) + model = torch.jit.freeze(model) # model = apply_jit_script(model) # model = apply_jit_trace(model, torch.from_numpy(np.expand_dims(dataset.get_input_array(), axis=0))) runner = PyTorchRunnerV2(model) From bc075b32f6d2a1ef7a09fa3905c5d5642884dff9 Mon Sep 17 00:00:00 2001 From: Marcel Wilnicki Date: Wed, 27 Mar 2024 12:18:21 +0100 Subject: [PATCH 18/35] wip --- .../unet_3d/kits_19/run.py | 24 ++----------------- 1 file changed, 2 insertions(+), 22 deletions(-) diff --git a/computer_vision/semantic_segmentation/unet_3d/kits_19/run.py b/computer_vision/semantic_segmentation/unet_3d/kits_19/run.py index a96722cbe..1990b6d1f 100644 --- a/computer_vision/semantic_segmentation/unet_3d/kits_19/run.py +++ b/computer_vision/semantic_segmentation/unet_3d/kits_19/run.py @@ -27,40 +27,20 @@ def run_single_pass(tf_runner, kits): def run_pytorch_fp(model_path, num_runs, timeout, kits_path): - from utils.pytorch import PyTorchRunnerV2, apply_jit_script, apply_jit_trace + from utils.pytorch import PyTorchRunnerV2 def run_single_pass(pytorch_runner, kits): output = pytorch_runner.run(1, torch.from_numpy(np.expand_dims(kits.get_input_array(), axis=0))) kits.submit_predictions(tf.convert_to_tensor(output.numpy())) dataset = KiTS19(dataset_dir_path=kits_path) - model = torch.jit.load(model_path, map_location=torch.device('cpu')) - model.eval() + model = torch.jit.load(model_path, map_location=torch.device('cpu')).eval() model = torch.jit.freeze(model) - # model = apply_jit_script(model) - # model = apply_jit_trace(model, torch.from_numpy(np.expand_dims(dataset.get_input_array(), axis=0))) runner = PyTorchRunnerV2(model) return run_model(run_single_pass, runner, dataset, 1, num_runs, timeout) -# def run_pytorch_fp(model_path, num_runs, timeout, kits_path): -# from utils.pytorch import PyTorchRunnerV2, apply_jit_script -# -# def run_single_pass(pytorch_runner, kits): -# output = pytorch_runner.run(1, torch.from_numpy(np.expand_dims(kits.get_input_array(), axis=0))) -# kits.submit_predictions(tf.convert_to_tensor(output.numpy())) -# -# dataset = KiTS19(dataset_dir_path=kits_path) -# dlrm.load_state_dict(torch.load(model_path)["state_dict"]) -# model = torch.load(model_path, map_location=torch.device('cpu')) -# model.eval() -# model = apply_jit_script(model) -# runner = PyTorchRunnerV2(model) -# -# return run_model(run_single_pass, runner, dataset, 1, num_runs, timeout) - - def run_tf_fp32(model_path, num_runs, timeout, kits_path, **kwargs): return run_tf_fp(model_path, num_runs, timeout, kits_path) From c9aacd82870ef896abbbfd04b023c0db2c33d1be Mon Sep 17 00:00:00 2001 From: Marcel Wilnicki Date: Thu, 4 Apr 2024 12:23:24 +0200 Subject: [PATCH 19/35] add tests --- tests/test_pytorch_models.py | 97 ++++++++++++++++++++++++------------ 1 file changed, 65 insertions(+), 32 deletions(-) diff --git a/tests/test_pytorch_models.py b/tests/test_pytorch_models.py index bfd1e070a..901b10a65 100644 --- a/tests/test_pytorch_models.py +++ b/tests/test_pytorch_models.py @@ -24,38 +24,38 @@ def run_process(wrapper, kwargs): return output -class LLaMA2(unittest.TestCase): - def setUp(self): - from natural_language_processing.text_generation.llama2.run import run_pytorch_fp32 - - url = "https://github.com/tloen/alpaca-lora/raw/main/alpaca_data.json" - self.dataset_path = pathlib.Path(get_downloads_path(), "alpaca_data.json") - if not self.dataset_path.exists(): - subprocess.run(f"wget -P {get_downloads_path()} {url}".split(), - check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) - - def wrapper(**kwargs): - kwargs["q"].put(run_pytorch_fp32(**kwargs)[0]) - - self.wrapper = wrapper - - @unittest.skipIf(psutil.virtual_memory().available / 1024 ** 3 < 100, "too little memory") - @unittest.skipUnless('_aio_profiler_print' in dir(torch._C), "Ampere optimized PyTorch required") - def test_llama2_7b(self): - f1_ref = 0.349 - acc = run_process(self.wrapper, - {"model_name": "meta-llama/Llama-2-7b-chat-hf", "batch_size": 1, "num_runs": 50, - "timeout": None, "dataset_path": self.dataset_path}) - self.assertTrue(acc["f1"] / f1_ref > 0.95) - - @unittest.skipIf(psutil.virtual_memory().available / 1024 ** 3 < 200, "too little memory") - @unittest.skipUnless('_aio_profiler_print' in dir(torch._C), "Ampere optimized PyTorch required") - def test_llama2_13b(self): - f1_ref = 0.195 - acc = run_process(self.wrapper, - {"model_name": "meta-llama/Llama-2-13b-chat-hf", "batch_size": 1, "num_runs": 50, - "timeout": None, "dataset_path": self.dataset_path}) - self.assertTrue(acc["f1"] / f1_ref > 0.95) +# class LLaMA2(unittest.TestCase): +# def setUp(self): +# from natural_language_processing.text_generation.llama2.run import run_pytorch_fp32 +# +# url = "https://github.com/tloen/alpaca-lora/raw/main/alpaca_data.json" +# self.dataset_path = pathlib.Path(get_downloads_path(), "alpaca_data.json") +# if not self.dataset_path.exists(): +# subprocess.run(f"wget -P {get_downloads_path()} {url}".split(), +# check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) +# +# def wrapper(**kwargs): +# kwargs["q"].put(run_pytorch_fp32(**kwargs)[0]) +# +# self.wrapper = wrapper +# +# @unittest.skipIf(psutil.virtual_memory().available / 1024 ** 3 < 100, "too little memory") +# @unittest.skipUnless('_aio_profiler_print' in dir(torch._C), "Ampere optimized PyTorch required") +# def test_llama2_7b(self): +# f1_ref = 0.349 +# acc = run_process(self.wrapper, +# {"model_name": "meta-llama/Llama-2-7b-chat-hf", "batch_size": 1, "num_runs": 50, +# "timeout": None, "dataset_path": self.dataset_path}) +# self.assertTrue(acc["f1"] / f1_ref > 0.95) +# +# @unittest.skipIf(psutil.virtual_memory().available / 1024 ** 3 < 200, "too little memory") +# @unittest.skipUnless('_aio_profiler_print' in dir(torch._C), "Ampere optimized PyTorch required") +# def test_llama2_13b(self): +# f1_ref = 0.195 +# acc = run_process(self.wrapper, +# {"model_name": "meta-llama/Llama-2-13b-chat-hf", "batch_size": 1, "num_runs": 50, +# "timeout": None, "dataset_path": self.dataset_path}) +# self.assertTrue(acc["f1"] / f1_ref > 0.95) class Alpaca(unittest.TestCase): @@ -176,6 +176,39 @@ def wrapper(**kwargs): self.assertTrue(acc["f1"] / f1_ref > 0.95) +class UNet3D(unittest.TestCase): + def setUp(self): + self.dataset_path = pathlib.Path(get_downloads_path(), "kits19_reduced.tar.gz") + if not self.dataset_path.exists(): + # url = os.environ.get("S3_URL_KITS19_REDUCED_DATASET") + url = "https://ampereaimodelzoo.s3.eu-central-1.amazonaws.com/kits19_reduced.tar.gz" + assert url is not None + subprocess.run(f"wget -P /tmp {url}".split(), + check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + subprocess.run(f"tar -xf /tmp/kits19_reduced.tar.gz -C {get_downloads_path()}".split(), + check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + subprocess.run("rm /tmp/criteo_preprocessed.tar.gz".split(), + check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + + self.model_path = pathlib.Path(get_downloads_path(), "3d_unet_kits_pytorch_fp32.ptc") + if not self.model_path.exists(): + # url = os.environ.get("S3_URL_UNET_KITS_PYTORCH_FP32") + url = "https://ampereaimodelzoo.s3.eu-central-1.amazonaws.com/3d_unet_kits_pytorch_fp32.ptc" + subprocess.run(f"wget -P /tmp {url}".split(), + check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + + def test_dlrm_debug(self): + from computer_vision.semantic_segmentation.unet_3d.kits_19.run import run_pytorch_fp32 + + def wrapper(**kwargs): + kwargs["q"].put(run_pytorch_fp32(**kwargs)[0]) + + auc_ref = 0.583 + acc = run_process(wrapper, {"model_path": self.model_path, "dataset_path": self.dataset_path, + "batch_size": 1, "num_runs": 30, "timeout": None, "debug": True}) + self.assertTrue(acc["auc"] / auc_ref > 0.95) + + def download_imagenet_maybe(): dataset_path = pathlib.Path(get_downloads_path(), "ILSVRC2012_onspecta") if not dataset_path.exists(): From e182f36a0450814f5364441cb50a2848c64b863b Mon Sep 17 00:00:00 2001 From: Marcel Wilnicki Date: Thu, 4 Apr 2024 12:38:04 +0200 Subject: [PATCH 20/35] wip --- tests/test_pytorch_models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_pytorch_models.py b/tests/test_pytorch_models.py index 901b10a65..2b46f8c25 100644 --- a/tests/test_pytorch_models.py +++ b/tests/test_pytorch_models.py @@ -178,7 +178,7 @@ def wrapper(**kwargs): class UNet3D(unittest.TestCase): def setUp(self): - self.dataset_path = pathlib.Path(get_downloads_path(), "kits19_reduced.tar.gz") + self.dataset_path = pathlib.Path(get_downloads_path(), "kits19") if not self.dataset_path.exists(): # url = os.environ.get("S3_URL_KITS19_REDUCED_DATASET") url = "https://ampereaimodelzoo.s3.eu-central-1.amazonaws.com/kits19_reduced.tar.gz" @@ -187,7 +187,7 @@ def setUp(self): check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) subprocess.run(f"tar -xf /tmp/kits19_reduced.tar.gz -C {get_downloads_path()}".split(), check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) - subprocess.run("rm /tmp/criteo_preprocessed.tar.gz".split(), + subprocess.run("rm /tmp/kits19_reduced.tar.gz".split(), check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) self.model_path = pathlib.Path(get_downloads_path(), "3d_unet_kits_pytorch_fp32.ptc") From 9461b9db83448fc8296be6d0a8fc7c15e2e5164b Mon Sep 17 00:00:00 2001 From: Marcel Wilnicki Date: Thu, 4 Apr 2024 12:40:02 +0200 Subject: [PATCH 21/35] wip --- tests/test_pytorch_models.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_pytorch_models.py b/tests/test_pytorch_models.py index 2b46f8c25..50637fc6d 100644 --- a/tests/test_pytorch_models.py +++ b/tests/test_pytorch_models.py @@ -178,6 +178,8 @@ def wrapper(**kwargs): class UNet3D(unittest.TestCase): def setUp(self): + print(get_downloads_path()) + quit() self.dataset_path = pathlib.Path(get_downloads_path(), "kits19") if not self.dataset_path.exists(): # url = os.environ.get("S3_URL_KITS19_REDUCED_DATASET") @@ -204,7 +206,7 @@ def wrapper(**kwargs): kwargs["q"].put(run_pytorch_fp32(**kwargs)[0]) auc_ref = 0.583 - acc = run_process(wrapper, {"model_path": self.model_path, "dataset_path": self.dataset_path, + acc = run_process(wrapper, {"model_path": self.model_path, "kits_path": self.dataset_path, "batch_size": 1, "num_runs": 30, "timeout": None, "debug": True}) self.assertTrue(acc["auc"] / auc_ref > 0.95) From 2ab67e03c0eee41ea85a2b23be2a1f290e51e780 Mon Sep 17 00:00:00 2001 From: Marcel Wilnicki Date: Thu, 4 Apr 2024 12:46:08 +0200 Subject: [PATCH 22/35] wip --- tests/test_pytorch_models.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/test_pytorch_models.py b/tests/test_pytorch_models.py index 50637fc6d..f36a4801f 100644 --- a/tests/test_pytorch_models.py +++ b/tests/test_pytorch_models.py @@ -176,10 +176,8 @@ def wrapper(**kwargs): self.assertTrue(acc["f1"] / f1_ref > 0.95) -class UNet3D(unittest.TestCase): +class UNET_KITS(unittest.TestCase): def setUp(self): - print(get_downloads_path()) - quit() self.dataset_path = pathlib.Path(get_downloads_path(), "kits19") if not self.dataset_path.exists(): # url = os.environ.get("S3_URL_KITS19_REDUCED_DATASET") @@ -199,7 +197,7 @@ def setUp(self): subprocess.run(f"wget -P /tmp {url}".split(), check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) - def test_dlrm_debug(self): + def test_unet_kits(self): from computer_vision.semantic_segmentation.unet_3d.kits_19.run import run_pytorch_fp32 def wrapper(**kwargs): From b611466462a1a217391fea77be915e63851b4d3e Mon Sep 17 00:00:00 2001 From: Marcel Wilnicki Date: Thu, 4 Apr 2024 12:56:13 +0200 Subject: [PATCH 23/35] wip --- tests/test_pytorch_models.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/test_pytorch_models.py b/tests/test_pytorch_models.py index f36a4801f..c3751e530 100644 --- a/tests/test_pytorch_models.py +++ b/tests/test_pytorch_models.py @@ -194,7 +194,7 @@ def setUp(self): if not self.model_path.exists(): # url = os.environ.get("S3_URL_UNET_KITS_PYTORCH_FP32") url = "https://ampereaimodelzoo.s3.eu-central-1.amazonaws.com/3d_unet_kits_pytorch_fp32.ptc" - subprocess.run(f"wget -P /tmp {url}".split(), + subprocess.run(f"wget -P {get_downloads_path()} {url}".split(), check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) def test_unet_kits(self): @@ -205,8 +205,11 @@ def wrapper(**kwargs): auc_ref = 0.583 acc = run_process(wrapper, {"model_path": self.model_path, "kits_path": self.dataset_path, - "batch_size": 1, "num_runs": 30, "timeout": None, "debug": True}) - self.assertTrue(acc["auc"] / auc_ref > 0.95) + "batch_size": 1, "num_runs": 15, "timeout": None, "debug": True}) + print(acc) + print(type(acc)) + quit() + # self.assertTrue(acc["auc"] / auc_ref > 0.95) def download_imagenet_maybe(): From f979209c4d4e046c5b75b24ccc53f8c5543fb4b2 Mon Sep 17 00:00:00 2001 From: Marcel Wilnicki Date: Thu, 4 Apr 2024 13:03:17 +0200 Subject: [PATCH 24/35] wip --- tests/test_pytorch_models.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/tests/test_pytorch_models.py b/tests/test_pytorch_models.py index c3751e530..011a4f209 100644 --- a/tests/test_pytorch_models.py +++ b/tests/test_pytorch_models.py @@ -203,13 +203,12 @@ def test_unet_kits(self): def wrapper(**kwargs): kwargs["q"].put(run_pytorch_fp32(**kwargs)[0]) - auc_ref = 0.583 + mean_kidney_acc, mean_tumor_acc = 0.927, 0.837 acc = run_process(wrapper, {"model_path": self.model_path, "kits_path": self.dataset_path, - "batch_size": 1, "num_runs": 15, "timeout": None, "debug": True}) - print(acc) - print(type(acc)) - quit() - # self.assertTrue(acc["auc"] / auc_ref > 0.95) + "batch_size": 1, "num_runs": 45, "timeout": None, "debug": True}) + + self.assertTrue(acc["mean_kidney_acc"] / mean_kidney_acc > 0.95) + self.assertTrue(acc["mean_tumor_acc"] / mean_tumor_acc > 0.95) def download_imagenet_maybe(): From ffadbf4999c2114b53507a13e3efcb1e205daced Mon Sep 17 00:00:00 2001 From: Marcel Wilnicki Date: Thu, 4 Apr 2024 13:12:17 +0200 Subject: [PATCH 25/35] wip --- tests/test_pytorch_models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_pytorch_models.py b/tests/test_pytorch_models.py index 011a4f209..5e6631938 100644 --- a/tests/test_pytorch_models.py +++ b/tests/test_pytorch_models.py @@ -205,7 +205,7 @@ def wrapper(**kwargs): mean_kidney_acc, mean_tumor_acc = 0.927, 0.837 acc = run_process(wrapper, {"model_path": self.model_path, "kits_path": self.dataset_path, - "batch_size": 1, "num_runs": 45, "timeout": None, "debug": True}) + "batch_size": 1, "num_runs": 200, "timeout": None, "debug": True}) self.assertTrue(acc["mean_kidney_acc"] / mean_kidney_acc > 0.95) self.assertTrue(acc["mean_tumor_acc"] / mean_tumor_acc > 0.95) From 9973ec10d578b2be9ca6f9f5b6ee5dc55946ebb9 Mon Sep 17 00:00:00 2001 From: Marcel Wilnicki Date: Thu, 4 Apr 2024 13:32:49 +0200 Subject: [PATCH 26/35] wip --- tests/test_pytorch_models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_pytorch_models.py b/tests/test_pytorch_models.py index 5e6631938..028d332a2 100644 --- a/tests/test_pytorch_models.py +++ b/tests/test_pytorch_models.py @@ -205,7 +205,7 @@ def wrapper(**kwargs): mean_kidney_acc, mean_tumor_acc = 0.927, 0.837 acc = run_process(wrapper, {"model_path": self.model_path, "kits_path": self.dataset_path, - "batch_size": 1, "num_runs": 200, "timeout": None, "debug": True}) + "batch_size": 1, "num_runs": 150, "timeout": None, "debug": True}) self.assertTrue(acc["mean_kidney_acc"] / mean_kidney_acc > 0.95) self.assertTrue(acc["mean_tumor_acc"] / mean_tumor_acc > 0.95) From f74996bd68bf18e057d5b63d8e2ce729f74cb052 Mon Sep 17 00:00:00 2001 From: Marcel Wilnicki Date: Thu, 4 Apr 2024 13:36:08 +0200 Subject: [PATCH 27/35] wip --- tests/test_pytorch_models.py | 64 ++++++++++++++++++------------------ 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/tests/test_pytorch_models.py b/tests/test_pytorch_models.py index 028d332a2..62a43405c 100644 --- a/tests/test_pytorch_models.py +++ b/tests/test_pytorch_models.py @@ -24,38 +24,38 @@ def run_process(wrapper, kwargs): return output -# class LLaMA2(unittest.TestCase): -# def setUp(self): -# from natural_language_processing.text_generation.llama2.run import run_pytorch_fp32 -# -# url = "https://github.com/tloen/alpaca-lora/raw/main/alpaca_data.json" -# self.dataset_path = pathlib.Path(get_downloads_path(), "alpaca_data.json") -# if not self.dataset_path.exists(): -# subprocess.run(f"wget -P {get_downloads_path()} {url}".split(), -# check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) -# -# def wrapper(**kwargs): -# kwargs["q"].put(run_pytorch_fp32(**kwargs)[0]) -# -# self.wrapper = wrapper -# -# @unittest.skipIf(psutil.virtual_memory().available / 1024 ** 3 < 100, "too little memory") -# @unittest.skipUnless('_aio_profiler_print' in dir(torch._C), "Ampere optimized PyTorch required") -# def test_llama2_7b(self): -# f1_ref = 0.349 -# acc = run_process(self.wrapper, -# {"model_name": "meta-llama/Llama-2-7b-chat-hf", "batch_size": 1, "num_runs": 50, -# "timeout": None, "dataset_path": self.dataset_path}) -# self.assertTrue(acc["f1"] / f1_ref > 0.95) -# -# @unittest.skipIf(psutil.virtual_memory().available / 1024 ** 3 < 200, "too little memory") -# @unittest.skipUnless('_aio_profiler_print' in dir(torch._C), "Ampere optimized PyTorch required") -# def test_llama2_13b(self): -# f1_ref = 0.195 -# acc = run_process(self.wrapper, -# {"model_name": "meta-llama/Llama-2-13b-chat-hf", "batch_size": 1, "num_runs": 50, -# "timeout": None, "dataset_path": self.dataset_path}) -# self.assertTrue(acc["f1"] / f1_ref > 0.95) +class LLaMA2(unittest.TestCase): + def setUp(self): + from natural_language_processing.text_generation.llama2.run import run_pytorch_fp32 + + url = "https://github.com/tloen/alpaca-lora/raw/main/alpaca_data.json" + self.dataset_path = pathlib.Path(get_downloads_path(), "alpaca_data.json") + if not self.dataset_path.exists(): + subprocess.run(f"wget -P {get_downloads_path()} {url}".split(), + check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + + def wrapper(**kwargs): + kwargs["q"].put(run_pytorch_fp32(**kwargs)[0]) + + self.wrapper = wrapper + + @unittest.skipIf(psutil.virtual_memory().available / 1024 ** 3 < 100, "too little memory") + @unittest.skipUnless('_aio_profiler_print' in dir(torch._C), "Ampere optimized PyTorch required") + def test_llama2_7b(self): + f1_ref = 0.349 + acc = run_process(self.wrapper, + {"model_name": "meta-llama/Llama-2-7b-chat-hf", "batch_size": 1, "num_runs": 50, + "timeout": None, "dataset_path": self.dataset_path}) + self.assertTrue(acc["f1"] / f1_ref > 0.95) + + @unittest.skipIf(psutil.virtual_memory().available / 1024 ** 3 < 200, "too little memory") + @unittest.skipUnless('_aio_profiler_print' in dir(torch._C), "Ampere optimized PyTorch required") + def test_llama2_13b(self): + f1_ref = 0.195 + acc = run_process(self.wrapper, + {"model_name": "meta-llama/Llama-2-13b-chat-hf", "batch_size": 1, "num_runs": 50, + "timeout": None, "dataset_path": self.dataset_path}) + self.assertTrue(acc["f1"] / f1_ref > 0.95) class Alpaca(unittest.TestCase): From 47e5bceb905b907c7142fd41dc73efdfa175759f Mon Sep 17 00:00:00 2001 From: Marcel Wilnicki Date: Thu, 4 Apr 2024 13:36:19 +0200 Subject: [PATCH 28/35] wip --- .github/workflows/test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 007d8a694..d625b4742 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -221,6 +221,8 @@ jobs: S3_URL_IMAGENET_DATASET_LABELS: ${{ secrets.S3_URL_IMAGENET_DATASET_LABELS }} S3_URL_COCO_DATASET: ${{ secrets.S3_URL_COCO_DATASET }} S3_URL_COCO_DATASET_ANNOTATIONS: ${{ secrets.S3_URL_COCO_DATASET_ANNOTATIONS }} + S3_URL_KITS19_REDUCED_DATASET: ${{ secrets.S3_URL_KITS19_REDUCED_DATASET }} + S3_URL_UNET_KITS_PYTORCH_FP32: ${{ secrets.S3_URL_UNET_KITS_PYTORCH_FP32 }} HF_HUB_TOKEN: ${{ secrets.HF_HUB_TOKEN }} steps: - name: Git checkout & pull submodules From 164ed88bb734264f6d42701b3e783e25267cd210 Mon Sep 17 00:00:00 2001 From: Marcel Wilnicki Date: Thu, 16 May 2024 10:45:10 +0200 Subject: [PATCH 29/35] wip --- computer_vision/semantic_segmentation/unet_3d/kits_19/run.py | 1 - 1 file changed, 1 deletion(-) diff --git a/computer_vision/semantic_segmentation/unet_3d/kits_19/run.py b/computer_vision/semantic_segmentation/unet_3d/kits_19/run.py index 77df58b2b..f5785f439 100644 --- a/computer_vision/semantic_segmentation/unet_3d/kits_19/run.py +++ b/computer_vision/semantic_segmentation/unet_3d/kits_19/run.py @@ -4,7 +4,6 @@ import torch import numpy as np import tensorflow as tf -from tensorflow.python.saved_model import tag_constants from utils.cv.kits import KiTS19 from utils.benchmark import run_model From fb7aaaf5b48e2ece09fce454efc4a1f9d9c5d8c3 Mon Sep 17 00:00:00 2001 From: Marcel Wilnicki Date: Thu, 16 May 2024 10:48:30 +0200 Subject: [PATCH 30/35] wip --- .../semantic_segmentation/unet_3d/kits_19/run.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/computer_vision/semantic_segmentation/unet_3d/kits_19/run.py b/computer_vision/semantic_segmentation/unet_3d/kits_19/run.py index f5785f439..bb2529380 100644 --- a/computer_vision/semantic_segmentation/unet_3d/kits_19/run.py +++ b/computer_vision/semantic_segmentation/unet_3d/kits_19/run.py @@ -1,13 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # Copyright (c) 2024, Ampere Computing LLC -import torch -import numpy as np -import tensorflow as tf - -from utils.cv.kits import KiTS19 -from utils.benchmark import run_model - try: from utils import misc # noqa @@ -49,7 +42,11 @@ def run_single_pass(tf_runner, kits): def run_pytorch_fp(model_path, num_runs, timeout, kits_path): + import torch + import numpy as np from utils.pytorch import PyTorchRunnerV2 + from utils.cv.kits import KiTS19 + from utils.benchmark import run_model def run_single_pass(pytorch_runner, kits): output = pytorch_runner.run(1, torch.from_numpy(np.expand_dims(kits.get_input_array(), axis=0))) From f390e47d833533cad69e060fb85269a971a10e5b Mon Sep 17 00:00:00 2001 From: Marcel Wilnicki Date: Thu, 16 May 2024 10:50:12 +0200 Subject: [PATCH 31/35] wip --- computer_vision/semantic_segmentation/unet_3d/kits_19/run.py | 1 + 1 file changed, 1 insertion(+) diff --git a/computer_vision/semantic_segmentation/unet_3d/kits_19/run.py b/computer_vision/semantic_segmentation/unet_3d/kits_19/run.py index bb2529380..807a93795 100644 --- a/computer_vision/semantic_segmentation/unet_3d/kits_19/run.py +++ b/computer_vision/semantic_segmentation/unet_3d/kits_19/run.py @@ -44,6 +44,7 @@ def run_single_pass(tf_runner, kits): def run_pytorch_fp(model_path, num_runs, timeout, kits_path): import torch import numpy as np + import tensorflow as tf from utils.pytorch import PyTorchRunnerV2 from utils.cv.kits import KiTS19 from utils.benchmark import run_model From 885d80bc79d28c8f5d09aa523f57676cc8c66bce Mon Sep 17 00:00:00 2001 From: Marcel Wilnicki Date: Thu, 16 May 2024 10:56:05 +0200 Subject: [PATCH 32/35] wip --- licensing/approved_urls.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/licensing/approved_urls.json b/licensing/approved_urls.json index 4f4136908..59f388223 100644 --- a/licensing/approved_urls.json +++ b/licensing/approved_urls.json @@ -38,6 +38,8 @@ "https://ampereaidevelopus.s3.amazonaws.com/releases/1.10.0/binaries/install_ampere_pytorch_u22_1_10_0.sh)", "https://ampereaimodelzoo.s3.eu-central-1.amazonaws.com/lookups_aml/banshee%40ampere_pytorch_1.10.0%40yolo_v8_s.json", "https://ampereaimodelzoo.s3.eu-central-1.amazonaws.com/sklearn2onnx/linear_regression_optimized.joblib", - "https://www.dropbox.com/s/ft54q1gi060vm2x/Task004_Hippocampus.zip?dl=1" + "https://www.dropbox.com/s/ft54q1gi060vm2x/Task004_Hippocampus.zip?dl=1", + "https://ampereaimodelzoo.s3.eu-central-1.amazonaws.com/3d_unet_kits_pytorch_fp32.ptc", + "https://ampereaimodelzoo.s3.eu-central-1.amazonaws.com/kits19_reduced.tar.gz" ] } \ No newline at end of file From 5a30eecbbadcf477ed5efb316a4811d83b7e68e2 Mon Sep 17 00:00:00 2001 From: Marcel Wilnicki Date: Thu, 16 May 2024 15:35:34 +0200 Subject: [PATCH 33/35] wip --- tests/test_pytorch_models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_pytorch_models.py b/tests/test_pytorch_models.py index f653ec7cd..1beffbcb8 100644 --- a/tests/test_pytorch_models.py +++ b/tests/test_pytorch_models.py @@ -256,7 +256,7 @@ def wrapper(**kwargs): mean_kidney_acc, mean_tumor_acc = 0.927, 0.837 acc = run_process(wrapper, {"model_path": self.model_path, "kits_path": self.dataset_path, - "batch_size": 1, "num_runs": 150, "timeout": None, "debug": True}) + "batch_size": 1, "num_runs": 1000, "timeout": 200, "debug": True}) self.assertTrue(acc["mean_kidney_acc"] / mean_kidney_acc > 0.95) self.assertTrue(acc["mean_tumor_acc"] / mean_tumor_acc > 0.95) From a2d715fba18a4941a65d4e71de6152884af995bb Mon Sep 17 00:00:00 2001 From: Marcel Wilnicki Date: Fri, 17 May 2024 15:40:55 +0200 Subject: [PATCH 34/35] wip --- tests/test_pytorch_models.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_pytorch_models.py b/tests/test_pytorch_models.py index 1beffbcb8..ddbe4f483 100644 --- a/tests/test_pytorch_models.py +++ b/tests/test_pytorch_models.py @@ -256,10 +256,10 @@ def wrapper(**kwargs): mean_kidney_acc, mean_tumor_acc = 0.927, 0.837 acc = run_process(wrapper, {"model_path": self.model_path, "kits_path": self.dataset_path, - "batch_size": 1, "num_runs": 1000, "timeout": 200, "debug": True}) + "batch_size": 1, "num_runs": 500, "timeout": 200, "debug": True}) - self.assertTrue(acc["mean_kidney_acc"] / mean_kidney_acc > 0.95) - self.assertTrue(acc["mean_tumor_acc"] / mean_tumor_acc > 0.95) + self.assertTrue(acc["mean_kidney_acc"] / mean_kidney_acc > 0.90) + self.assertTrue(acc["mean_tumor_acc"] / mean_tumor_acc > 0.80) def download_imagenet_maybe(): From ac799fc70f465b51dcbbc686f2f582136ac47b53 Mon Sep 17 00:00:00 2001 From: Marcel Wilnicki Date: Thu, 23 May 2024 16:30:25 +0200 Subject: [PATCH 35/35] wip --- licensing/approved_urls.json | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/licensing/approved_urls.json b/licensing/approved_urls.json index 59f388223..4f4136908 100644 --- a/licensing/approved_urls.json +++ b/licensing/approved_urls.json @@ -38,8 +38,6 @@ "https://ampereaidevelopus.s3.amazonaws.com/releases/1.10.0/binaries/install_ampere_pytorch_u22_1_10_0.sh)", "https://ampereaimodelzoo.s3.eu-central-1.amazonaws.com/lookups_aml/banshee%40ampere_pytorch_1.10.0%40yolo_v8_s.json", "https://ampereaimodelzoo.s3.eu-central-1.amazonaws.com/sklearn2onnx/linear_regression_optimized.joblib", - "https://www.dropbox.com/s/ft54q1gi060vm2x/Task004_Hippocampus.zip?dl=1", - "https://ampereaimodelzoo.s3.eu-central-1.amazonaws.com/3d_unet_kits_pytorch_fp32.ptc", - "https://ampereaimodelzoo.s3.eu-central-1.amazonaws.com/kits19_reduced.tar.gz" + "https://www.dropbox.com/s/ft54q1gi060vm2x/Task004_Hippocampus.zip?dl=1" ] } \ No newline at end of file