From 0d618488d48f558e6173d2fdad033fa3c92e68ca Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 25 Dec 2024 12:15:26 +0000 Subject: [PATCH 1/6] Fix coco2014 sample ids path --- script/get-dataset-coco2014/customize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/get-dataset-coco2014/customize.py b/script/get-dataset-coco2014/customize.py index b03e75049..3f80e76f0 100644 --- a/script/get-dataset-coco2014/customize.py +++ b/script/get-dataset-coco2014/customize.py @@ -24,7 +24,7 @@ def postprocess(i): env = i['env'] if env.get('CM_GENERATE_SAMPLE_ID', '') == "yes": env['CM_COCO2014_SAMPLE_ID_PATH'] = os.path.join( - os.getcwd(), 'install', 'sample_ids.txt') + os.getcwd(), 'sample_ids.txt') print(env['CM_COCO2014_SAMPLE_ID_PATH']) if env.get('CM_DATASET_CALIBRATION', '') == "no": env['CM_DATASET_PATH_ROOT'] = os.getcwd() From ef8ab61b6fc39d73f3265be62162fafa3c2c18f0 Mon Sep 17 00:00:00 2001 From: Arjun Date: Fri, 27 Dec 2024 11:40:20 +0000 Subject: [PATCH 2/6] Fixes for podman, #33 --- automation/script/module.py | 51 ++++++++++++++---------- automation/script/module_misc.py | 10 ++++- script/get-ml-model-rgat/_cm.yaml | 4 +- script/get-ml-model-rgat/customize.py | 10 +++-- script/run-docker-container/_cm.yaml | 1 + script/run-docker-container/customize.py | 3 ++ 6 files changed, 51 insertions(+), 28 deletions(-) diff --git a/automation/script/module.py b/automation/script/module.py index eda2920c7..07b9bfe3e 100644 --- a/automation/script/module.py +++ b/automation/script/module.py @@ -507,28 +507,7 @@ def _run(self, i): if os.environ.get(key, '') != '' and env.get(key, '') == '': env[key] = os.environ[key] - # Check path/input/output in input and pass to env - for key in self.input_flags_converted_to_tmp_env: - value = i.get(key, '').strip() - if value != '': - env['CM_TMP_' + key.upper()] = value - - for key in self.input_flags_converted_to_env: - value = i.get( - key, - '').strip() if isinstance( - i.get( - key, - ''), - str) else i.get( - key, - '') - if value: - env[f"CM_{key.upper()}"] = value - - r = update_env_with_values(env) - if r['return'] > 0: - return r + r = self._update_env_from_input(env, i) ####################################################################### # Check if we want to skip cache (either by skip_cache or by fake_run) @@ -2294,6 +2273,34 @@ def _run(self, i): return rr + + ########################################################################## + def _update_env_from_input(self, env, i): + # Check path/input/output in input and pass to env + for key in self.input_flags_converted_to_tmp_env: + value = i.get(key, '').strip() + if value != '': + env['CM_TMP_' + key.upper()] = value + + for key in self.input_flags_converted_to_env: + value = i.get( + key, + '').strip() if isinstance( + i.get( + key, + ''), + str) else i.get( + key, + '') + if value: + env[f"CM_{key.upper()}"] = value + + r = update_env_with_values(env) + if r['return'] > 0: + return r + + return {'return': 0} + ########################################################################## def _fix_cache_paths(self, env): cm_repos_path = os.environ.get( diff --git a/automation/script/module_misc.py b/automation/script/module_misc.py index 5de7f0402..d9f37992b 100644 --- a/automation/script/module_misc.py +++ b/automation/script/module_misc.py @@ -2071,6 +2071,14 @@ def docker(i): continue ''' + r = script_automation._update_env_from_input(env, i) + if r['return'] > 0: + return r + + # mount outdirname path + if env.get('CM_OUTDIRNAME', '') != '': + mounts.append(f"""{env['CM_OUTDIRNAME']}:{env['CM_OUTDIRNAME']}""") + # Check if need to update/map/mount inputs and env r = process_inputs({'run_cmd_arc': i_run_cmd_arc, 'docker_settings': docker_settings, @@ -2409,7 +2417,7 @@ def docker(i): print(final_run_cmd) print('') - docker_recreate_image = 'yes' if not norecreate_docker_image else 'no' + docker_recreate_image = 'yes' if str(norecreate_docker_image).lower() not in ["yes", "true", "1"] else 'no' if i.get('docker_push_image', '') in ['True', True, 'yes']: env['CM_DOCKER_PUSH_IMAGE'] = 'yes' diff --git a/script/get-ml-model-rgat/_cm.yaml b/script/get-ml-model-rgat/_cm.yaml index bcec6df1c..54e5c119c 100644 --- a/script/get-ml-model-rgat/_cm.yaml +++ b/script/get-ml-model-rgat/_cm.yaml @@ -3,13 +3,13 @@ automation_alias: script automation_uid: 5b4e0237da074764 cache: true category: AI/ML models +docker: + fake_run_deps: True env: CM_ML_MODEL: RGAT CM_ML_MODEL_DATASET: ICBH input_mapping: checkpoint: RGAT_CHECKPOINT_PATH - download_path: CM_DOWNLOAD_PATH - to: CM_DOWNLOAD_PATH new_env_keys: - CM_ML_MODEL_* - CM_ML_MODEL_RGAT_CHECKPOINT_PATH diff --git a/script/get-ml-model-rgat/customize.py b/script/get-ml-model-rgat/customize.py index d920a8711..56fd2ecf7 100644 --- a/script/get-ml-model-rgat/customize.py +++ b/script/get-ml-model-rgat/customize.py @@ -7,10 +7,15 @@ def preprocess(i): os_info = i['os_info'] env = i['env'] + download_dir = env.get('CM_OUTDIRNAME', '') + path = env.get('RGAT_CHECKPOINT_PATH', '').strip() if path == '' or not os.path.exists(path): - env['CM_TMP_REQUIRE_DOWNLOAD'] = 'yes' + if download_dir != '' and os.path.exists(os.path.join(download_dir, "RGAT", "RGAT.pt")): + env['RGAT_CHECKPOINT_PATH'] = os.path.join(download_dir, "RGAT", "RGAT.pt") + else: + env['CM_TMP_REQUIRE_DOWNLOAD'] = 'yes' return {'return': 0} @@ -20,8 +25,7 @@ def postprocess(i): env = i['env'] if env.get('CM_ML_MODEL_RGAT_CHECKPOINT_PATH', '') == '': - env['CM_ML_MODEL_RGAT_CHECKPOINT_PATH'] = env.get( - 'RGAT_CHECKPOINT_PATH', os.path.join(env['RGAT_DIR_PATH'], "RGAT.pt")) + env['CM_ML_MODEL_RGAT_CHECKPOINT_PATH'] = env['RGAT_CHECKPOINT_PATH'] if env.get('CM_ML_MODEL_PATH', '') == '': env['CM_ML_MODEL_PATH'] = env['CM_ML_MODEL_RGAT_CHECKPOINT_PATH'] diff --git a/script/run-docker-container/_cm.yaml b/script/run-docker-container/_cm.yaml index 5135070b9..8a04e5b43 100644 --- a/script/run-docker-container/_cm.yaml +++ b/script/run-docker-container/_cm.yaml @@ -68,3 +68,4 @@ prehook_deps: CM_DOCKER_CONTAINER_ID: - on tags: build,docker,image +- tags: get,docker diff --git a/script/run-docker-container/customize.py b/script/run-docker-container/customize.py index 869386f91..cea1ccea3 100644 --- a/script/run-docker-container/customize.py +++ b/script/run-docker-container/customize.py @@ -167,6 +167,9 @@ def postprocess(i): if env.get('CM_DOCKER_EXTRA_RUN_ARGS', '') != '': run_opts += env['CM_DOCKER_EXTRA_RUN_ARGS'] + if env.get('CM_CONTAINER_TOOL', '') == 'podman' and env.get('CM_PODMAN_MAP_USER_ID','').lower() not in ["no", "0", "false"]: + run_opts += " --userns=keep-id" + if env.get('CM_DOCKER_PORT_MAPS', []): for ports in env['CM_DOCKER_PORT_MAPS']: port_map_cmds.append(ports) From b93b4da0d2ef1ff8d404f64908245b4c4f14e2cc Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Fri, 27 Dec 2024 11:41:03 +0000 Subject: [PATCH 3/6] [Automated Commit] Format Codebase --- automation/script/module.py | 2 +- automation/script/module_misc.py | 3 ++- script/get-ml-model-rgat/customize.py | 6 ++++-- script/run-docker-container/customize.py | 3 ++- 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/automation/script/module.py b/automation/script/module.py index 07b9bfe3e..f37dc9830 100644 --- a/automation/script/module.py +++ b/automation/script/module.py @@ -2273,8 +2273,8 @@ def _run(self, i): return rr - ########################################################################## + def _update_env_from_input(self, env, i): # Check path/input/output in input and pass to env for key in self.input_flags_converted_to_tmp_env: diff --git a/automation/script/module_misc.py b/automation/script/module_misc.py index d9f37992b..cf6e94597 100644 --- a/automation/script/module_misc.py +++ b/automation/script/module_misc.py @@ -2417,7 +2417,8 @@ def docker(i): print(final_run_cmd) print('') - docker_recreate_image = 'yes' if str(norecreate_docker_image).lower() not in ["yes", "true", "1"] else 'no' + docker_recreate_image = 'yes' if str(norecreate_docker_image).lower() not in [ + "yes", "true", "1"] else 'no' if i.get('docker_push_image', '') in ['True', True, 'yes']: env['CM_DOCKER_PUSH_IMAGE'] = 'yes' diff --git a/script/get-ml-model-rgat/customize.py b/script/get-ml-model-rgat/customize.py index 56fd2ecf7..5d190f33f 100644 --- a/script/get-ml-model-rgat/customize.py +++ b/script/get-ml-model-rgat/customize.py @@ -12,8 +12,10 @@ def preprocess(i): path = env.get('RGAT_CHECKPOINT_PATH', '').strip() if path == '' or not os.path.exists(path): - if download_dir != '' and os.path.exists(os.path.join(download_dir, "RGAT", "RGAT.pt")): - env['RGAT_CHECKPOINT_PATH'] = os.path.join(download_dir, "RGAT", "RGAT.pt") + if download_dir != '' and os.path.exists( + os.path.join(download_dir, "RGAT", "RGAT.pt")): + env['RGAT_CHECKPOINT_PATH'] = os.path.join( + download_dir, "RGAT", "RGAT.pt") else: env['CM_TMP_REQUIRE_DOWNLOAD'] = 'yes' diff --git a/script/run-docker-container/customize.py b/script/run-docker-container/customize.py index cea1ccea3..2157d595f 100644 --- a/script/run-docker-container/customize.py +++ b/script/run-docker-container/customize.py @@ -167,7 +167,8 @@ def postprocess(i): if env.get('CM_DOCKER_EXTRA_RUN_ARGS', '') != '': run_opts += env['CM_DOCKER_EXTRA_RUN_ARGS'] - if env.get('CM_CONTAINER_TOOL', '') == 'podman' and env.get('CM_PODMAN_MAP_USER_ID','').lower() not in ["no", "0", "false"]: + if env.get('CM_CONTAINER_TOOL', '') == 'podman' and env.get( + 'CM_PODMAN_MAP_USER_ID', '').lower() not in ["no", "0", "false"]: run_opts += " --userns=keep-id" if env.get('CM_DOCKER_PORT_MAPS', []): From 685acbae603686e20283e15d9c7db1b92bc8c39b Mon Sep 17 00:00:00 2001 From: Arjun Date: Fri, 27 Dec 2024 11:47:35 +0000 Subject: [PATCH 4/6] Fixed rgat checkpoint path --- script/get-ml-model-rgat/customize.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/script/get-ml-model-rgat/customize.py b/script/get-ml-model-rgat/customize.py index 5d190f33f..3021321a1 100644 --- a/script/get-ml-model-rgat/customize.py +++ b/script/get-ml-model-rgat/customize.py @@ -26,6 +26,9 @@ def postprocess(i): env = i['env'] + if env.get('RGAT_CHECKPOINT_PATH', '') == '': + env['RGAT_CHECKPOINT_PATH'] = os.path.join(env['RGAT_DIR_PATH'], "RGAT.pt")) + if env.get('CM_ML_MODEL_RGAT_CHECKPOINT_PATH', '') == '': env['CM_ML_MODEL_RGAT_CHECKPOINT_PATH'] = env['RGAT_CHECKPOINT_PATH'] From 9913f6e86f1d3af33cf4b07352d292e6da5b0fe1 Mon Sep 17 00:00:00 2001 From: Arjun Date: Fri, 27 Dec 2024 11:50:36 +0000 Subject: [PATCH 5/6] Fixed rgat checkpoint path --- script/get-ml-model-rgat/customize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/get-ml-model-rgat/customize.py b/script/get-ml-model-rgat/customize.py index 3021321a1..00bec2630 100644 --- a/script/get-ml-model-rgat/customize.py +++ b/script/get-ml-model-rgat/customize.py @@ -27,7 +27,7 @@ def postprocess(i): env = i['env'] if env.get('RGAT_CHECKPOINT_PATH', '') == '': - env['RGAT_CHECKPOINT_PATH'] = os.path.join(env['RGAT_DIR_PATH'], "RGAT.pt")) + env['RGAT_CHECKPOINT_PATH'] = os.path.join(env['RGAT_DIR_PATH'], "RGAT.pt") if env.get('CM_ML_MODEL_RGAT_CHECKPOINT_PATH', '') == '': env['CM_ML_MODEL_RGAT_CHECKPOINT_PATH'] = env['RGAT_CHECKPOINT_PATH'] From 72d4a85f90b5a0e83d0e7c044c21d5c6d1e4a9af Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Fri, 27 Dec 2024 11:50:59 +0000 Subject: [PATCH 6/6] [Automated Commit] Format Codebase --- script/get-ml-model-rgat/customize.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/script/get-ml-model-rgat/customize.py b/script/get-ml-model-rgat/customize.py index 00bec2630..3cda5a7a8 100644 --- a/script/get-ml-model-rgat/customize.py +++ b/script/get-ml-model-rgat/customize.py @@ -27,7 +27,8 @@ def postprocess(i): env = i['env'] if env.get('RGAT_CHECKPOINT_PATH', '') == '': - env['RGAT_CHECKPOINT_PATH'] = os.path.join(env['RGAT_DIR_PATH'], "RGAT.pt") + env['RGAT_CHECKPOINT_PATH'] = os.path.join( + env['RGAT_DIR_PATH'], "RGAT.pt") if env.get('CM_ML_MODEL_RGAT_CHECKPOINT_PATH', '') == '': env['CM_ML_MODEL_RGAT_CHECKPOINT_PATH'] = env['RGAT_CHECKPOINT_PATH']