From 9ceafe43865e05169d8f717d35c4ee25f4dec887 Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Wed, 23 Apr 2025 12:47:34 +0530 Subject: [PATCH 1/2] fixes for path issues --- script/benchmark-program/customize.py | 3 +-- script/compile-program/run.bat | 8 ++++---- script/compile-program/run.sh | 10 +++++----- script/extract-file/customize.py | 4 ++-- script/extract-file/run.bat | 10 +++++----- script/get-preprocessed-dataset-criteo/customize.py | 2 +- .../preprocess_multihot.sh | 10 +++++----- 7 files changed, 23 insertions(+), 24 deletions(-) diff --git a/script/benchmark-program/customize.py b/script/benchmark-program/customize.py index a355e8248..de0fac475 100644 --- a/script/benchmark-program/customize.py +++ b/script/benchmark-program/customize.py @@ -43,8 +43,7 @@ def preprocess(i): if env.get('MLC_RUN_DIR', '') == '': env['MLC_RUN_DIR'] = os.getcwd() - env['MLC_RUN_CMD'] = MLC_RUN_PREFIX + ' ' + os.path.join( - env['MLC_RUN_DIR'], env['MLC_BIN_NAME']) + ' ' + env['MLC_RUN_SUFFIX'] + env['MLC_RUN_CMD'] = f"""{MLC_RUN_PREFIX} {q}{os.path.join(env['MLC_RUN_DIR'], env['MLC_BIN_NAME'])}{q} {env['MLC_RUN_SUFFIX']}""" x = env.get('MLC_RUN_PREFIX0', '') if x != '': diff --git a/script/compile-program/run.bat b/script/compile-program/run.bat index 8a9e5436d..52bbaba75 100644 --- a/script/compile-program/run.bat +++ b/script/compile-program/run.bat @@ -16,20 +16,20 @@ echo. echo Compiling source files ... echo. -if not exist %RUN_DIR% mkdir %RUN_DIR% +if not exist "%RUN_DIR%" mkdir "%RUN_DIR%" -cd %MLC_SOURCE_FOLDER_PATH% +cd "%MLC_SOURCE_FOLDER_PATH%" IF %ERRORLEVEL% NEQ 0 EXIT %ERRORLEVEL% if not "%MLC_C_SOURCE_FILES%" == "" ( echo %MLC_C_COMPILER_WITH_PATH% %MLC_C_COMPILER_FLAGS% %MLC_C_INCLUDE_PATH% %MLC_C_SOURCE_FILES% %MLC_LD_LIBRARY_PATH% %LDCFLAGS% %MLC_C_COMPILER_FLAG_OUTPUT%"%RUN_DIR%\%BIN_NAME%" - "%MLC_C_COMPILER_WITH_PATH%" %MLC_C_COMPILER_FLAGS% %MLC_C_INCLUDE_PATH% %MLC_C_SOURCE_FILES% %MLC_LD_LIBRARY_PATH% %LDCFLAGS% %MLC_C_COMPILER_FLAG_OUTPUT%"%RUN_DIR%\%BIN_NAME%" + "%MLC_C_COMPILER_WITH_PATH%" %MLC_C_COMPILER_FLAGS% "%MLC_C_INCLUDE_PATH%" %MLC_C_SOURCE_FILES% "%MLC_LD_LIBRARY_PATH%" %LDCFLAGS% %MLC_C_COMPILER_FLAG_OUTPUT%"%RUN_DIR%\%BIN_NAME%" IF %ERRORLEVEL% NEQ 0 EXIT %ERRORLEVEL% ) if not "%MLC_CXX_SOURCE_FILES%" == "" ( echo %MLC_CXX_COMPILER_WITH_PATH% %MLC_CXX_SOURCE_FILES% %MLC_CXX_COMPILER_FLAGS% %MLC_CPLUS_INCLUDE_PATH% %MLC_LD_LIBRARY_PATH% %LDCXXFLAGS% %MLC_CXX_COMPILER_FLAG_OUTPUT%"%RUN_DIR%\%BIN_NAME%" - "%MLC_CXX_COMPILER_WITH_PATH%" %MLC_CXX_SOURCE_FILES% %MLC_CXX_COMPILER_FLAGS% %MLC_CPLUS_INCLUDE_PATH% %MLC_LD_LIBRARY_PATH% %LDCXXFLAGS% %MLC_CXX_COMPILER_FLAG_OUTPUT%"%RUN_DIR%\%BIN_NAME%" + "%MLC_CXX_COMPILER_WITH_PATH%" %MLC_CXX_SOURCE_FILES% %MLC_CXX_COMPILER_FLAGS% "%MLC_CPLUS_INCLUDE_PATH%" "%MLC_LD_LIBRARY_PATH%" %LDCXXFLAGS% %MLC_CXX_COMPILER_FLAG_OUTPUT%"%RUN_DIR%\%BIN_NAME%" IF %ERRORLEVEL% NEQ 0 EXIT %ERRORLEVEL% ) diff --git a/script/compile-program/run.sh b/script/compile-program/run.sh index 68045945d..171b99070 100644 --- a/script/compile-program/run.sh +++ b/script/compile-program/run.sh @@ -3,16 +3,16 @@ # Compile BIN_NAME=${MLC_BIN_NAME:-run.out} -RUN_DIR=${MLC_RUN_DIR:-.} +RUN_DIR="${MLC_RUN_DIR:-.}" echo "RUN_DIR=$RUN_DIR" if [[ ${MLC_SKIP_RECOMPILE} == "yes" ]]; then - if [ -f ${RUN_DIR}/${BIN_NAME} ]; then + if [ -f "${RUN_DIR}/${BIN_NAME}" ]; then exit 0 fi fi -rm -f ${RUN_DIR}/${BIN_NAME} +rm -f "${RUN_DIR}/${BIN_NAME}" if [ -z "${MLC_SOURCE_FOLDER_PATH}" ]; then echo "No source directory (MLC_SOURCE_FOLDER_PATH} specified" @@ -28,13 +28,13 @@ echo "" echo "Checking compiler version ..." echo "" -${MLC_C_COMPILER_WITH_PATH} ${MLC_C_COMPILER_FLAG_VERSION} +"${MLC_C_COMPILER_WITH_PATH}" ${MLC_C_COMPILER_FLAG_VERSION} echo "" echo "Compiling source files ..." echo "" -cd ${MLC_SOURCE_FOLDER_PATH} +cd "${MLC_SOURCE_FOLDER_PATH}" test $? -eq 0 || exit 1 IFS=';' read -ra FILES <<< "${MLC_C_SOURCE_FILES}" diff --git a/script/extract-file/customize.py b/script/extract-file/customize.py index f6f041dbc..89b57c927 100644 --- a/script/extract-file/customize.py +++ b/script/extract-file/customize.py @@ -117,10 +117,10 @@ def preprocess(i): y = '"' if ' ' in extract_to_folder else '' # env['MLC_EXTRACT_TOOL_OPTIONS'] = ' --one-top-level='+ env['MLC_EXTRACT_TO_FOLDER'] + env.get('MLC_EXTRACT_TOOL_OPTIONS', '') - env['MLC_EXTRACT_TOOL_OPTIONS'] = ' -C ' + y + extract_to_folder + \ + env['MLC_EXTRACT_TOOL_OPTIONS'] = ' -C ' + y + q + extract_to_folder + q + \ y + ' ' + env.get('MLC_EXTRACT_TOOL_OPTIONS', '') env['MLC_EXTRACT_PRE_CMD'] = 'mkdir ' + x + ' ' + \ - y + extract_to_folder + y + ' ' + xsep + ' ' + y + q + extract_to_folder + q + y + ' ' + xsep + ' ' env['MLC_EXTRACT_EXTRACTED_FILENAME'] = extract_to_folder elif 'unzip' in env['MLC_EXTRACT_TOOL']: diff --git a/script/extract-file/run.bat b/script/extract-file/run.bat index 2a2727965..bfc066533 100644 --- a/script/extract-file/run.bat +++ b/script/extract-file/run.bat @@ -12,10 +12,10 @@ if exist "%MLC_EXTRACT_EXTRACTED_FILENAME%" ( echo. echo %MLC_EXTRACT_EXTRACTED_CHECKSUM_CMD% - cmd /c %MLC_EXTRACT_EXTRACTED_CHECKSUM_CMD% + cmd /c "%MLC_EXTRACT_EXTRACTED_CHECKSUM_CMD%" IF !ERRORLEVEL! NEQ 0 ( set require_extract=1 - del /Q %MLC_EXTRACT_EXTRACTED_FILENAME% + del /Q "%MLC_EXTRACT_EXTRACTED_FILENAME%" ) ) @@ -23,17 +23,17 @@ if "!require_extract!" == "1" ( if not "%MLC_EXTRACT_CMD0%" == "" ( echo. echo %MLC_EXTRACT_CMD0% - cmd /c %MLC_EXTRACT_CMD0% + cmd /c "%MLC_EXTRACT_CMD0%" IF !ERRORLEVEL! NEQ 0 EXIT 1 ) echo. echo %MLC_EXTRACT_CMD% - cmd /c %MLC_EXTRACT_CMD% + cmd /c "%MLC_EXTRACT_CMD%" IF !ERRORLEVEL! NEQ 0 EXIT 1 echo. echo %MLC_EXTRACT_EXTRACTED_CHECKSUM_CMD% - cmd /c %MLC_EXTRACT_EXTRACTED_CHECKSUM_CMD% + cmd /c "%MLC_EXTRACT_EXTRACTED_CHECKSUM_CMD%" IF !ERRORLEVEL! NEQ 0 EXIT 1 ) diff --git a/script/get-preprocessed-dataset-criteo/customize.py b/script/get-preprocessed-dataset-criteo/customize.py index 723e209bf..668152284 100644 --- a/script/get-preprocessed-dataset-criteo/customize.py +++ b/script/get-preprocessed-dataset-criteo/customize.py @@ -32,7 +32,7 @@ def preprocess(i): "recommendation_v2", "torchrec_dlrm", "scripts") - env['MLC_RUN_CMD'] = f'cd {run_dir} && bash ./process_Criteo_1TB_Click_Logs_dataset.sh {dataset_path} {tmp_dir} {output_dir} ' + env['MLC_RUN_CMD'] = f"""cd '{run_dir}' && bash ./process_Criteo_1TB_Click_Logs_dataset.sh '{dataset_path}' '{tmp_dir}' '{output_dir}' """ print("Using MLCommons Training source from '" + env['MLC_MLPERF_TRAINING_SOURCE'] + "'") diff --git a/script/get-preprocessed-dataset-criteo/preprocess_multihot.sh b/script/get-preprocessed-dataset-criteo/preprocess_multihot.sh index dadf7566c..e38ecd87c 100644 --- a/script/get-preprocessed-dataset-criteo/preprocess_multihot.sh +++ b/script/get-preprocessed-dataset-criteo/preprocess_multihot.sh @@ -1,9 +1,9 @@ #!/bin/bash -cd ${MLC_MLPERF_TRAINING_SOURCE}/recommendation_v2_torchrec_dlrm/ -${MLC_PYTHON_BIN_WITH_PATH} materialize_synthetic_multihot_dataset.py \ - --in_memory_binary_criteo_path $PREPROCESSED_CRITEO_1TB_CLICK_LOGS_DATASET_PATH \ - --output_path $MATERIALIZED_DATASET_PATH \ +cd "${MLC_MLPERF_TRAINING_SOURCE}/recommendation_v2_torchrec_dlrm/" +"${MLC_PYTHON_BIN_WITH_PATH}" "materialize_synthetic_multihot_dataset.py \ + --in_memory_binary_criteo_path ${PREPROCESSED_CRITEO_1TB_CLICK_LOGS_DATASET_PATH} \ + --output_path ${MATERIALIZED_DATASET_PATH} \ --num_embeddings_per_feature 40000000,39060,17295,7424,20265,3,7122,1543,63,40000000,3067956,405282,10,2209,11938,155,4,976,14,40000000,40000000,40000000,590152,12973,108,36 \ --multi_hot_sizes 3,2,1,2,6,1,1,1,1,7,3,8,1,6,9,5,1,1,1,12,100,27,10,3,1,1 \ - --multi_hot_distribution_type uniform + --multi_hot_distribution_type uniform" test $? -eq 0 || exit $? From a6316bcbde6b65bae33cf5b3b45e3147592440bf Mon Sep 17 00:00:00 2001 From: anandhu-eng Date: Wed, 23 Apr 2025 12:55:37 +0530 Subject: [PATCH 2/2] test commit --- script/extract-file/customize.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/script/extract-file/customize.py b/script/extract-file/customize.py index 89b57c927..f6f041dbc 100644 --- a/script/extract-file/customize.py +++ b/script/extract-file/customize.py @@ -117,10 +117,10 @@ def preprocess(i): y = '"' if ' ' in extract_to_folder else '' # env['MLC_EXTRACT_TOOL_OPTIONS'] = ' --one-top-level='+ env['MLC_EXTRACT_TO_FOLDER'] + env.get('MLC_EXTRACT_TOOL_OPTIONS', '') - env['MLC_EXTRACT_TOOL_OPTIONS'] = ' -C ' + y + q + extract_to_folder + q + \ + env['MLC_EXTRACT_TOOL_OPTIONS'] = ' -C ' + y + extract_to_folder + \ y + ' ' + env.get('MLC_EXTRACT_TOOL_OPTIONS', '') env['MLC_EXTRACT_PRE_CMD'] = 'mkdir ' + x + ' ' + \ - y + q + extract_to_folder + q + y + ' ' + xsep + ' ' + y + extract_to_folder + y + ' ' + xsep + ' ' env['MLC_EXTRACT_EXTRACTED_FILENAME'] = extract_to_folder elif 'unzip' in env['MLC_EXTRACT_TOOL']: