diff --git a/plugins/nf-azure/src/main/nextflow/cloud/azure/file/AzBashLib.groovy b/plugins/nf-azure/src/main/nextflow/cloud/azure/file/AzBashLib.groovy index 0653da019a..1755d1f806 100644 --- a/plugins/nf-azure/src/main/nextflow/cloud/azure/file/AzBashLib.groovy +++ b/plugins/nf-azure/src/main/nextflow/cloud/azure/file/AzBashLib.groovy @@ -73,25 +73,26 @@ class AzBashLib extends BashFunLib { azcopy cp "$name" "$target/$name?$AZ_SAS" --block-blob-tier $AZCOPY_BLOCK_BLOB_TIER --block-size-mb $AZCOPY_BLOCK_SIZE_MB fi } - + nxf_az_download() { local source=$1 local target=$2 local basedir=$(dirname $2) - local ret mkdir -p "$basedir" - - ret=$(azcopy cp "$source?$AZ_SAS" "$target" 2>&1) || { - ## if fails check if it was trying to download a directory - mkdir -p $target - azcopy cp "$source/*?$AZ_SAS" "$target" --recursive >/dev/null || { - rm -rf $target + + # Try to download as file first, let azcopy handle the detection + if ! azcopy cp "$source?$AZ_SAS" "$target"; then + # If failed, remove any partial target and try as directory + rm -rf "$target" + mkdir -p "$target" + if ! azcopy cp "$source/*?$AZ_SAS" "$target" --recursive; then + rm -rf "$target" >&2 echo "Unable to download path: $source" exit 1 - } - } + fi + fi } - '''.stripIndent(true) + '''.stripIndent() } String render() { diff --git a/plugins/nf-azure/src/test/nextflow/cloud/azure/batch/AzFileCopyStrategyTest.groovy b/plugins/nf-azure/src/test/nextflow/cloud/azure/batch/AzFileCopyStrategyTest.groovy index 436fcb2ce9..fd7b77f3ec 100644 --- a/plugins/nf-azure/src/test/nextflow/cloud/azure/batch/AzFileCopyStrategyTest.groovy +++ b/plugins/nf-azure/src/test/nextflow/cloud/azure/batch/AzFileCopyStrategyTest.groovy @@ -179,18 +179,19 @@ class AzFileCopyStrategyTest extends Specification { local source=$1 local target=$2 local basedir=$(dirname $2) - local ret mkdir -p "$basedir" - ret=$(azcopy cp "$source?$AZ_SAS" "$target" 2>&1) || { - ## if fails check if it was trying to download a directory - mkdir -p $target - azcopy cp "$source/*?$AZ_SAS" "$target" --recursive >/dev/null || { - rm -rf $target + # Try to download as file first, let azcopy handle the detection + if ! azcopy cp "$source?$AZ_SAS" "$target"; then + # If failed, remove any partial target and try as directory + rm -rf "$target" + mkdir -p "$target" + if ! azcopy cp "$source/*?$AZ_SAS" "$target" --recursive; then + rm -rf "$target" >&2 echo "Unable to download path: $source" exit 1 - } - } + fi + fi } '''.stripIndent(true) @@ -316,18 +317,19 @@ class AzFileCopyStrategyTest extends Specification { local source=$1 local target=$2 local basedir=$(dirname $2) - local ret mkdir -p "$basedir" - ret=$(azcopy cp "$source?$AZ_SAS" "$target" 2>&1) || { - ## if fails check if it was trying to download a directory - mkdir -p $target - azcopy cp "$source/*?$AZ_SAS" "$target" --recursive >/dev/null || { - rm -rf $target + # Try to download as file first, let azcopy handle the detection + if ! azcopy cp "$source?$AZ_SAS" "$target"; then + # If failed, remove any partial target and try as directory + rm -rf "$target" + mkdir -p "$target" + if ! azcopy cp "$source/*?$AZ_SAS" "$target" --recursive; then + rm -rf "$target" >&2 echo "Unable to download path: $source" exit 1 - } - } + fi + fi } '''.stripIndent(true) @@ -477,18 +479,19 @@ class AzFileCopyStrategyTest extends Specification { local source=$1 local target=$2 local basedir=$(dirname $2) - local ret mkdir -p "$basedir" - ret=$(azcopy cp "$source?$AZ_SAS" "$target" 2>&1) || { - ## if fails check if it was trying to download a directory - mkdir -p $target - azcopy cp "$source/*?$AZ_SAS" "$target" --recursive >/dev/null || { - rm -rf $target + # Try to download as file first, let azcopy handle the detection + if ! azcopy cp "$source?$AZ_SAS" "$target"; then + # If failed, remove any partial target and try as directory + rm -rf "$target" + mkdir -p "$target" + if ! azcopy cp "$source/*?$AZ_SAS" "$target" --recursive; then + rm -rf "$target" >&2 echo "Unable to download path: $source" exit 1 - } - } + fi + fi } '''.stripIndent(true) diff --git a/plugins/nf-azure/src/test/nextflow/cloud/azure/file/AzBashLibTest.groovy b/plugins/nf-azure/src/test/nextflow/cloud/azure/file/AzBashLibTest.groovy index b300da8f5e..4155e77fbf 100644 --- a/plugins/nf-azure/src/test/nextflow/cloud/azure/file/AzBashLibTest.groovy +++ b/plugins/nf-azure/src/test/nextflow/cloud/azure/file/AzBashLibTest.groovy @@ -38,18 +38,19 @@ class AzBashLibTest extends Specification { local source=$1 local target=$2 local basedir=$(dirname $2) - local ret mkdir -p "$basedir" - ret=$(azcopy cp "$source?$AZ_SAS" "$target" 2>&1) || { - ## if fails check if it was trying to download a directory - mkdir -p $target - azcopy cp "$source/*?$AZ_SAS" "$target" --recursive >/dev/null || { - rm -rf $target + # Try to download as file first, let azcopy handle the detection + if ! azcopy cp "$source?$AZ_SAS" "$target"; then + # If failed, remove any partial target and try as directory + rm -rf "$target" + mkdir -p "$target" + if ! azcopy cp "$source/*?$AZ_SAS" "$target" --recursive; then + rm -rf "$target" >&2 echo "Unable to download path: $source" exit 1 - } - } + fi + fi } '''.stripIndent() } @@ -140,18 +141,19 @@ class AzBashLibTest extends Specification { local source=$1 local target=$2 local basedir=$(dirname $2) - local ret mkdir -p "$basedir" - - ret=$(azcopy cp "$source?$AZ_SAS" "$target" 2>&1) || { - ## if fails check if it was trying to download a directory - mkdir -p $target - azcopy cp "$source/*?$AZ_SAS" "$target" --recursive >/dev/null || { - rm -rf $target + + # Try to download as file first, let azcopy handle the detection + if ! azcopy cp "$source?$AZ_SAS" "$target"; then + # If failed, remove any partial target and try as directory + rm -rf "$target" + mkdir -p "$target" + if ! azcopy cp "$source/*?$AZ_SAS" "$target" --recursive; then + rm -rf "$target" >&2 echo "Unable to download path: $source" exit 1 - } - } + fi + fi } '''.stripIndent() } @@ -242,18 +244,19 @@ class AzBashLibTest extends Specification { local source=$1 local target=$2 local basedir=$(dirname $2) - local ret mkdir -p "$basedir" - ret=$(azcopy cp "$source?$AZ_SAS" "$target" 2>&1) || { - ## if fails check if it was trying to download a directory - mkdir -p $target - azcopy cp "$source/*?$AZ_SAS" "$target" --recursive >/dev/null || { - rm -rf $target + # Try to download as file first, let azcopy handle the detection + if ! azcopy cp "$source?$AZ_SAS" "$target"; then + # If failed, remove any partial target and try as directory + rm -rf "$target" + mkdir -p "$target" + if ! azcopy cp "$source/*?$AZ_SAS" "$target" --recursive; then + rm -rf "$target" >&2 echo "Unable to download path: $source" exit 1 - } - } + fi + fi } '''.stripIndent() } diff --git a/plugins/nf-azure/src/test/nextflow/executor/BashWrapperBuilderWithAzTest.groovy b/plugins/nf-azure/src/test/nextflow/executor/BashWrapperBuilderWithAzTest.groovy index 1969345af5..d6a380fb60 100644 --- a/plugins/nf-azure/src/test/nextflow/executor/BashWrapperBuilderWithAzTest.groovy +++ b/plugins/nf-azure/src/test/nextflow/executor/BashWrapperBuilderWithAzTest.groovy @@ -77,15 +77,14 @@ class BashWrapperBuilderWithAzTest extends Specification { nxf_az_download() { local source=$1 local target=$2 - local basedir=$(dirname $2) - local ret + local basedir=$(dirname "$target") mkdir -p "$basedir" - ret=$(azcopy cp "$source?$AZ_SAS" "$target" 2>&1) || { - ## if fails check if it was trying to download a directory - mkdir -p $target - azcopy cp "$source/*?$AZ_SAS" "$target" --recursive >/dev/null || { - rm -rf $target + if ! azcopy cp "$source?$AZ_SAS" "$target"; then + # If failed, remove any partial target and try as directory + rm -rf "$target"; mkdir -p "$target" + if ! azcopy cp "$source/*?$AZ_SAS" "$target" --recursive; then + rm -rf "$target" >&2 echo "Unable to download path: $source" exit 1 } @@ -216,15 +215,14 @@ class BashWrapperBuilderWithAzTest extends Specification { nxf_az_download() { local source=$1 local target=$2 - local basedir=$(dirname $2) - local ret + local basedir=$(dirname "$target") mkdir -p "$basedir" - ret=$(azcopy cp "$source?$AZ_SAS" "$target" 2>&1) || { - ## if fails check if it was trying to download a directory - mkdir -p $target - azcopy cp "$source/*?$AZ_SAS" "$target" --recursive >/dev/null || { - rm -rf $target + if ! azcopy cp "$source?$AZ_SAS" "$target"; then + # If failed, remove any partial target and try as directory + rm -rf "$target"; mkdir -p "$target" + if ! azcopy cp "$source/*?$AZ_SAS" "$target" --recursive; then + rm -rf "$target" >&2 echo "Unable to download path: $source" exit 1 }