From 8ffe60b5bb0aa1fa248fa944751be65a8d78c9e5 Mon Sep 17 00:00:00 2001
From: adamrtalbot <12817534+adamrtalbot@users.noreply.github.com>
Date: Fri, 6 Jun 2025 20:17:00 +0100
Subject: [PATCH 1/7] Prevents azcopy hiding error messages by redirection

Fixes #6158

Signed-off-by: adamrtalbot <12817534+adamrtalbot@users.noreply.github.com>
---
 .../cloud/azure/file/AzBashLib.groovy         | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/plugins/nf-azure/src/main/nextflow/cloud/azure/file/AzBashLib.groovy b/plugins/nf-azure/src/main/nextflow/cloud/azure/file/AzBashLib.groovy
index 0653da019a..2da893b09e 100644
--- a/plugins/nf-azure/src/main/nextflow/cloud/azure/file/AzBashLib.groovy
+++ b/plugins/nf-azure/src/main/nextflow/cloud/azure/file/AzBashLib.groovy
@@ -70,7 +70,7 @@ class AzBashLib extends BashFunLib<AzBashLib> {
                 azcopy cp "$name" "$target/$dir_name?$AZ_SAS" --recursive --block-blob-tier $AZCOPY_BLOCK_BLOB_TIER --block-size-mb $AZCOPY_BLOCK_SIZE_MB
               fi
             else
-              azcopy cp "$name" "$target/$name?$AZ_SAS" --block-blob-tier $AZCOPY_BLOCK_BLOB_TIER --block-size-mb $AZCOPY_BLOCK_SIZE_MB
+              azcopy cp --output-level quiet "$name" "$target/$name?$AZ_SAS" --block-blob-tier $AZCOPY_BLOCK_BLOB_TIER --block-size-mb $AZCOPY_BLOCK_SIZE_MB
             fi
         }
         
@@ -78,18 +78,19 @@ class AzBashLib extends BashFunLib<AzBashLib> {
             local source=$1
             local target=$2
             local basedir=$(dirname $2)
-            local ret
             mkdir -p "$basedir"
         
-            ret=$(azcopy cp "$source?$AZ_SAS" "$target" 2>&1) || {
-                ## if fails check if it was trying to download a directory
-                mkdir -p $target
-                azcopy cp "$source/*?$AZ_SAS" "$target" --recursive >/dev/null || {
-                    rm -rf $target
+            # Try to download as file first, let azcopy handle the detection
+            if ! azcopy cp --output-level quiet "$source?$AZ_SAS" "$target"; then
+                # If failed, remove any partial target and try as directory
+                rm -rf "$target"
+                mkdir -p "$target"
+                if ! azcopy cp --output-level quiet"$source/*?$AZ_SAS" "$target" --recursive; then
+                    rm -rf "$target"
                     >&2 echo "Unable to download path: $source"
                     exit 1
-                }
-            }
+                fi
+            fi
         }
         '''.stripIndent(true)
     }

From 0053f945463cf2d80bfd33ffc5354e02c0337525 Mon Sep 17 00:00:00 2001
From: adamrtalbot <12817534+adamrtalbot@users.noreply.github.com>
Date: Thu, 12 Jun 2025 11:38:19 +0100
Subject: [PATCH 2/7] Handle azcopy failure better

Signed-off-by: adamrtalbot <12817534+adamrtalbot@users.noreply.github.com>
---
 .../azure/batch/AzFileCopyStrategyTest.groovy | 34 ++++++------
 .../cloud/azure/file/AzBashLibTest.groovy     | 55 ++++++++++---------
 .../BashWrapperBuilderWithAzTest.groovy       | 24 ++++----
 3 files changed, 59 insertions(+), 54 deletions(-)

diff --git a/plugins/nf-azure/src/test/nextflow/cloud/azure/batch/AzFileCopyStrategyTest.groovy b/plugins/nf-azure/src/test/nextflow/cloud/azure/batch/AzFileCopyStrategyTest.groovy
index 436fcb2ce9..f8c5069d33 100644
--- a/plugins/nf-azure/src/test/nextflow/cloud/azure/batch/AzFileCopyStrategyTest.groovy
+++ b/plugins/nf-azure/src/test/nextflow/cloud/azure/batch/AzFileCopyStrategyTest.groovy
@@ -316,18 +316,19 @@ class AzFileCopyStrategyTest extends Specification {
                     local source=$1
                     local target=$2
                     local basedir=$(dirname $2)
-                    local ret
                     mkdir -p "$basedir"
                 
-                    ret=$(azcopy cp "$source?$AZ_SAS" "$target" 2>&1) || {
-                        ## if fails check if it was trying to download a directory
-                        mkdir -p $target
-                        azcopy cp "$source/*?$AZ_SAS" "$target" --recursive >/dev/null || {
-                            rm -rf $target
+                    # Try to download as file first, let azcopy handle the detection
+                    if ! azcopy cp "$source?$AZ_SAS" "$target"; then
+                        # If failed, remove any partial target and try as directory
+                        rm -rf "$target"
+                        mkdir -p "$target"
+                        if ! azcopy cp "$source/*?$AZ_SAS" "$target" --recursive; then
+                            rm -rf "$target"
                             >&2 echo "Unable to download path: $source"
                             exit 1
-                        }
-                    }
+                        fi
+                    fi
                 }
 
                 '''.stripIndent(true)
@@ -477,18 +478,19 @@ class AzFileCopyStrategyTest extends Specification {
                         local source=$1
                         local target=$2
                         local basedir=$(dirname $2)
-                        local ret
                         mkdir -p "$basedir"
                     
-                        ret=$(azcopy cp "$source?$AZ_SAS" "$target" 2>&1) || {
-                            ## if fails check if it was trying to download a directory
-                            mkdir -p $target
-                            azcopy cp "$source/*?$AZ_SAS" "$target" --recursive >/dev/null || {
-                                rm -rf $target
+                        # Try to download as file first, let azcopy handle the detection
+                        if ! azcopy cp "$source?$AZ_SAS" "$target"; then
+                            # If failed, remove any partial target and try as directory
+                            rm -rf "$target"
+                            mkdir -p "$target"
+                            if ! azcopy cp "$source/*?$AZ_SAS" "$target" --recursive; then
+                                rm -rf "$target"
                                 >&2 echo "Unable to download path: $source"
                                 exit 1
-                            }
-                        }
+                            fi
+                        fi
                     }
 
                     '''.stripIndent(true)
diff --git a/plugins/nf-azure/src/test/nextflow/cloud/azure/file/AzBashLibTest.groovy b/plugins/nf-azure/src/test/nextflow/cloud/azure/file/AzBashLibTest.groovy
index b300da8f5e..fed371da92 100644
--- a/plugins/nf-azure/src/test/nextflow/cloud/azure/file/AzBashLibTest.groovy
+++ b/plugins/nf-azure/src/test/nextflow/cloud/azure/file/AzBashLibTest.groovy
@@ -38,18 +38,19 @@ class AzBashLibTest extends Specification {
                 local source=$1
                 local target=$2
                 local basedir=$(dirname $2)
-                local ret
                 mkdir -p "$basedir"
 
-                ret=$(azcopy cp "$source?$AZ_SAS" "$target" 2>&1) || {
-                    ## if fails check if it was trying to download a directory
-                    mkdir -p $target
-                    azcopy cp "$source/*?$AZ_SAS" "$target" --recursive >/dev/null || {
-                        rm -rf $target
+                # Try to download as file first, let azcopy handle the detection
+                if ! azcopy cp "$source?$AZ_SAS" "$target"; then
+                    # If failed, remove any partial target and try as directory
+                    rm -rf "$target"
+                    mkdir -p "$target"
+                    if ! azcopy cp "$source/*?$AZ_SAS" "$target" --recursive; then
+                        rm -rf "$target"
                         >&2 echo "Unable to download path: $source"
                         exit 1
-                    }
-                }
+                    fi
+                fi
             }
             '''.stripIndent()
     }
@@ -136,22 +137,23 @@ class AzBashLibTest extends Specification {
                 fi
             }
             
-            nxf_az_download() {
+                        nxf_az_download() {
                 local source=$1
                 local target=$2
                 local basedir=$(dirname $2)
-                local ret
                 mkdir -p "$basedir"
-            
-                ret=$(azcopy cp "$source?$AZ_SAS" "$target" 2>&1) || {
-                    ## if fails check if it was trying to download a directory
-                    mkdir -p $target
-                    azcopy cp "$source/*?$AZ_SAS" "$target" --recursive >/dev/null || {
-                        rm -rf $target
+
+                # Try to download as file first, let azcopy handle the detection
+                if ! azcopy cp "$source?$AZ_SAS" "$target"; then
+                    # If failed, remove any partial target and try as directory
+                    rm -rf "$target"
+                    mkdir -p "$target"
+                    if ! azcopy cp "$source/*?$AZ_SAS" "$target" --recursive; then
+                        rm -rf "$target"
                         >&2 echo "Unable to download path: $source"
                         exit 1
-                    }
-                }
+                    fi
+                fi
             }
             '''.stripIndent()
     }
@@ -242,18 +244,19 @@ class AzBashLibTest extends Specification {
                 local source=$1
                 local target=$2
                 local basedir=$(dirname $2)
-                local ret
                 mkdir -p "$basedir"
             
-                ret=$(azcopy cp "$source?$AZ_SAS" "$target" 2>&1) || {
-                    ## if fails check if it was trying to download a directory
-                    mkdir -p $target
-                    azcopy cp "$source/*?$AZ_SAS" "$target" --recursive >/dev/null || {
-                        rm -rf $target
+                # Try to download as file first, let azcopy handle the detection
+                if ! azcopy cp "$source?$AZ_SAS" "$target"; then
+                    # If failed, remove any partial target and try as directory
+                    rm -rf "$target"
+                    mkdir -p "$target"
+                    if ! azcopy cp "$source/*?$AZ_SAS" "$target" --recursive; then
+                        rm -rf "$target"
                         >&2 echo "Unable to download path: $source"
                         exit 1
-                    }
-                }
+                    fi
+                fi
             }
             '''.stripIndent()
     }
diff --git a/plugins/nf-azure/src/test/nextflow/executor/BashWrapperBuilderWithAzTest.groovy b/plugins/nf-azure/src/test/nextflow/executor/BashWrapperBuilderWithAzTest.groovy
index 1969345af5..fe8b11c5ed 100644
--- a/plugins/nf-azure/src/test/nextflow/executor/BashWrapperBuilderWithAzTest.groovy
+++ b/plugins/nf-azure/src/test/nextflow/executor/BashWrapperBuilderWithAzTest.groovy
@@ -78,14 +78,14 @@ class BashWrapperBuilderWithAzTest extends Specification {
                 local source=$1
                 local target=$2
                 local basedir=$(dirname $2)
-                local ret
+                # local ret removed
                 mkdir -p "$basedir"
             
-                ret=$(azcopy cp "$source?$AZ_SAS" "$target" 2>&1) || {
-                    ## if fails check if it was trying to download a directory
-                    mkdir -p $target
-                    azcopy cp "$source/*?$AZ_SAS" "$target" --recursive >/dev/null || {
-                        rm -rf $target
+                if ! azcopy cp "$source?$AZ_SAS" "$target"; then
+                    # If failed, remove any partial target and try as directory
+                    rm -rf "$target"; mkdir -p "$target"
+                    if ! azcopy cp "$source/*?$AZ_SAS" "$target" --recursive; then
+                        rm -rf "$target"
                         >&2 echo "Unable to download path: $source"
                         exit 1
                     }
@@ -217,14 +217,14 @@ class BashWrapperBuilderWithAzTest extends Specification {
                 local source=$1
                 local target=$2
                 local basedir=$(dirname $2)
-                local ret
+                # local ret removed
                 mkdir -p "$basedir"
             
-                ret=$(azcopy cp "$source?$AZ_SAS" "$target" 2>&1) || {
-                    ## if fails check if it was trying to download a directory
-                    mkdir -p $target
-                    azcopy cp "$source/*?$AZ_SAS" "$target" --recursive >/dev/null || {
-                        rm -rf $target
+                if ! azcopy cp "$source?$AZ_SAS" "$target"; then
+                    # If failed, remove any partial target and try as directory
+                    rm -rf "$target"; mkdir -p "$target"
+                    if ! azcopy cp "$source/*?$AZ_SAS" "$target" --recursive; then
+                        rm -rf "$target"
                         >&2 echo "Unable to download path: $source"
                         exit 1
                     }

From 6cc90209addf6cc293e71bcbc85e4f88af9d3e33 Mon Sep 17 00:00:00 2001
From: adamrtalbot <12817534+adamrtalbot@users.noreply.github.com>
Date: Mon, 16 Jun 2025 11:48:07 +0100
Subject: [PATCH 3/7] Remove --output-level quiet from azcopy calls

Signed-off-by: adamrtalbot <12817534+adamrtalbot@users.noreply.github.com>
---
 .../nextflow/cloud/azure/file/AzBashLib.groovy  | 12 ++++++------
 .../azure/batch/AzFileCopyStrategyTest.groovy   | 17 +++++++++--------
 2 files changed, 15 insertions(+), 14 deletions(-)

diff --git a/plugins/nf-azure/src/main/nextflow/cloud/azure/file/AzBashLib.groovy b/plugins/nf-azure/src/main/nextflow/cloud/azure/file/AzBashLib.groovy
index 2da893b09e..1755d1f806 100644
--- a/plugins/nf-azure/src/main/nextflow/cloud/azure/file/AzBashLib.groovy
+++ b/plugins/nf-azure/src/main/nextflow/cloud/azure/file/AzBashLib.groovy
@@ -70,29 +70,29 @@ class AzBashLib extends BashFunLib<AzBashLib> {
                 azcopy cp "$name" "$target/$dir_name?$AZ_SAS" --recursive --block-blob-tier $AZCOPY_BLOCK_BLOB_TIER --block-size-mb $AZCOPY_BLOCK_SIZE_MB
               fi
             else
-              azcopy cp --output-level quiet "$name" "$target/$name?$AZ_SAS" --block-blob-tier $AZCOPY_BLOCK_BLOB_TIER --block-size-mb $AZCOPY_BLOCK_SIZE_MB
+              azcopy cp "$name" "$target/$name?$AZ_SAS" --block-blob-tier $AZCOPY_BLOCK_BLOB_TIER --block-size-mb $AZCOPY_BLOCK_SIZE_MB
             fi
         }
-        
+
         nxf_az_download() {
             local source=$1
             local target=$2
             local basedir=$(dirname $2)
             mkdir -p "$basedir"
-        
+
             # Try to download as file first, let azcopy handle the detection
-            if ! azcopy cp --output-level quiet "$source?$AZ_SAS" "$target"; then
+            if ! azcopy cp "$source?$AZ_SAS" "$target"; then
                 # If failed, remove any partial target and try as directory
                 rm -rf "$target"
                 mkdir -p "$target"
-                if ! azcopy cp --output-level quiet"$source/*?$AZ_SAS" "$target" --recursive; then
+                if ! azcopy cp "$source/*?$AZ_SAS" "$target" --recursive; then
                     rm -rf "$target"
                     >&2 echo "Unable to download path: $source"
                     exit 1
                 fi
             fi
         }
-        '''.stripIndent(true)
+        '''.stripIndent()
     }
 
     String render() {
diff --git a/plugins/nf-azure/src/test/nextflow/cloud/azure/batch/AzFileCopyStrategyTest.groovy b/plugins/nf-azure/src/test/nextflow/cloud/azure/batch/AzFileCopyStrategyTest.groovy
index f8c5069d33..fd7b77f3ec 100644
--- a/plugins/nf-azure/src/test/nextflow/cloud/azure/batch/AzFileCopyStrategyTest.groovy
+++ b/plugins/nf-azure/src/test/nextflow/cloud/azure/batch/AzFileCopyStrategyTest.groovy
@@ -179,18 +179,19 @@ class AzFileCopyStrategyTest extends Specification {
                     local source=$1
                     local target=$2
                     local basedir=$(dirname $2)
-                    local ret
                     mkdir -p "$basedir"
                 
-                    ret=$(azcopy cp "$source?$AZ_SAS" "$target" 2>&1) || {
-                        ## if fails check if it was trying to download a directory
-                        mkdir -p $target
-                        azcopy cp "$source/*?$AZ_SAS" "$target" --recursive >/dev/null || {
-                            rm -rf $target
+                    # Try to download as file first, let azcopy handle the detection
+                    if ! azcopy cp "$source?$AZ_SAS" "$target"; then
+                        # If failed, remove any partial target and try as directory
+                        rm -rf "$target"
+                        mkdir -p "$target"
+                        if ! azcopy cp "$source/*?$AZ_SAS" "$target" --recursive; then
+                            rm -rf "$target"
                             >&2 echo "Unable to download path: $source"
                             exit 1
-                        }
-                    }
+                        fi
+                    fi
                 }
                 
                 '''.stripIndent(true)

From 5a0ebdba19ca6c5a60fe8093919cdb7c4577759c Mon Sep 17 00:00:00 2001
From: adamrtalbot <12817534+adamrtalbot@users.noreply.github.com>
Date: Mon, 16 Jun 2025 11:48:26 +0100
Subject: [PATCH 4/7] fix test indentation

Signed-off-by: adamrtalbot <12817534+adamrtalbot@users.noreply.github.com>
---
 .../src/test/nextflow/cloud/azure/file/AzBashLibTest.groovy     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/plugins/nf-azure/src/test/nextflow/cloud/azure/file/AzBashLibTest.groovy b/plugins/nf-azure/src/test/nextflow/cloud/azure/file/AzBashLibTest.groovy
index fed371da92..4155e77fbf 100644
--- a/plugins/nf-azure/src/test/nextflow/cloud/azure/file/AzBashLibTest.groovy
+++ b/plugins/nf-azure/src/test/nextflow/cloud/azure/file/AzBashLibTest.groovy
@@ -137,7 +137,7 @@ class AzBashLibTest extends Specification {
                 fi
             }
             
-                        nxf_az_download() {
+            nxf_az_download() {
                 local source=$1
                 local target=$2
                 local basedir=$(dirname $2)

From cfdf0cb236224024ea84a66db71601ab3c4cb6ea Mon Sep 17 00:00:00 2001
From: adamrtalbot <12817534+adamrtalbot@users.noreply.github.com>
Date: Mon, 16 Jun 2025 14:24:26 +0100
Subject: [PATCH 5/7] Replace azcopy cp with azcopy sync in bash lib

This seems to work with both files and directories with no issue. I tested it with:

```bash
make pack && ./build/releases/nextflow-25.05.0-edge-dist run nf-core/rnaseq -r 3.19.0 -profile test --outdir results -c azure.config -w az://work/
BUILD_PACK=1 ./gradlew pack
```

And it worked fine?

Signed-off-by: adamrtalbot <12817534+adamrtalbot@users.noreply.github.com>
---
 .../cloud/azure/file/AzBashLib.groovy         | 14 +----
 .../azure/batch/AzFileCopyStrategyTest.groovy | 48 ++++-------------
 .../cloud/azure/file/AzBashLibTest.groovy     | 54 ++++++-------------
 .../BashWrapperBuilderWithAzTest.groovy       | 28 +++-------
 4 files changed, 33 insertions(+), 111 deletions(-)

diff --git a/plugins/nf-azure/src/main/nextflow/cloud/azure/file/AzBashLib.groovy b/plugins/nf-azure/src/main/nextflow/cloud/azure/file/AzBashLib.groovy
index 1755d1f806..31ce2774d2 100644
--- a/plugins/nf-azure/src/main/nextflow/cloud/azure/file/AzBashLib.groovy
+++ b/plugins/nf-azure/src/main/nextflow/cloud/azure/file/AzBashLib.groovy
@@ -77,20 +77,10 @@ class AzBashLib extends BashFunLib<AzBashLib> {
         nxf_az_download() {
             local source=$1
             local target=$2
-            local basedir=$(dirname $2)
+            local basedir=$(dirname "$target")
             mkdir -p "$basedir"
 
-            # Try to download as file first, let azcopy handle the detection
-            if ! azcopy cp "$source?$AZ_SAS" "$target"; then
-                # If failed, remove any partial target and try as directory
-                rm -rf "$target"
-                mkdir -p "$target"
-                if ! azcopy cp "$source/*?$AZ_SAS" "$target" --recursive; then
-                    rm -rf "$target"
-                    >&2 echo "Unable to download path: $source"
-                    exit 1
-                fi
-            fi
+            azcopy sync "$source?$AZ_SAS" "$target" --recursive
         }
         '''.stripIndent()
     }
diff --git a/plugins/nf-azure/src/test/nextflow/cloud/azure/batch/AzFileCopyStrategyTest.groovy b/plugins/nf-azure/src/test/nextflow/cloud/azure/batch/AzFileCopyStrategyTest.groovy
index fd7b77f3ec..7830bb30ed 100644
--- a/plugins/nf-azure/src/test/nextflow/cloud/azure/batch/AzFileCopyStrategyTest.groovy
+++ b/plugins/nf-azure/src/test/nextflow/cloud/azure/batch/AzFileCopyStrategyTest.groovy
@@ -178,20 +178,10 @@ class AzFileCopyStrategyTest extends Specification {
                 nxf_az_download() {
                     local source=$1
                     local target=$2
-                    local basedir=$(dirname $2)
+                    local basedir=$(dirname "$target")
                     mkdir -p "$basedir"
-                
-                    # Try to download as file first, let azcopy handle the detection
-                    if ! azcopy cp "$source?$AZ_SAS" "$target"; then
-                        # If failed, remove any partial target and try as directory
-                        rm -rf "$target"
-                        mkdir -p "$target"
-                        if ! azcopy cp "$source/*?$AZ_SAS" "$target" --recursive; then
-                            rm -rf "$target"
-                            >&2 echo "Unable to download path: $source"
-                            exit 1
-                        fi
-                    fi
+
+                    azcopy sync "$source?$AZ_SAS" "$target" --recursive
                 }
                 
                 '''.stripIndent(true)
@@ -316,20 +306,10 @@ class AzFileCopyStrategyTest extends Specification {
                 nxf_az_download() {
                     local source=$1
                     local target=$2
-                    local basedir=$(dirname $2)
+                    local basedir=$(dirname "$target")
                     mkdir -p "$basedir"
-                
-                    # Try to download as file first, let azcopy handle the detection
-                    if ! azcopy cp "$source?$AZ_SAS" "$target"; then
-                        # If failed, remove any partial target and try as directory
-                        rm -rf "$target"
-                        mkdir -p "$target"
-                        if ! azcopy cp "$source/*?$AZ_SAS" "$target" --recursive; then
-                            rm -rf "$target"
-                            >&2 echo "Unable to download path: $source"
-                            exit 1
-                        fi
-                    fi
+
+                    azcopy sync "$source?$AZ_SAS" "$target" --recursive
                 }
 
                 '''.stripIndent(true)
@@ -478,20 +458,10 @@ class AzFileCopyStrategyTest extends Specification {
                     nxf_az_download() {
                         local source=$1
                         local target=$2
-                        local basedir=$(dirname $2)
+                        local basedir=$(dirname "$target")
                         mkdir -p "$basedir"
-                    
-                        # Try to download as file first, let azcopy handle the detection
-                        if ! azcopy cp "$source?$AZ_SAS" "$target"; then
-                            # If failed, remove any partial target and try as directory
-                            rm -rf "$target"
-                            mkdir -p "$target"
-                            if ! azcopy cp "$source/*?$AZ_SAS" "$target" --recursive; then
-                                rm -rf "$target"
-                                >&2 echo "Unable to download path: $source"
-                                exit 1
-                            fi
-                        fi
+
+                        azcopy sync "$source?$AZ_SAS" "$target" --recursive
                     }
 
                     '''.stripIndent(true)
diff --git a/plugins/nf-azure/src/test/nextflow/cloud/azure/file/AzBashLibTest.groovy b/plugins/nf-azure/src/test/nextflow/cloud/azure/file/AzBashLibTest.groovy
index 4155e77fbf..00eb1c1c22 100644
--- a/plugins/nf-azure/src/test/nextflow/cloud/azure/file/AzBashLibTest.groovy
+++ b/plugins/nf-azure/src/test/nextflow/cloud/azure/file/AzBashLibTest.groovy
@@ -37,20 +37,10 @@ class AzBashLibTest extends Specification {
             nxf_az_download() {
                 local source=$1
                 local target=$2
-                local basedir=$(dirname $2)
+                local basedir=$(dirname "$target")
                 mkdir -p "$basedir"
 
-                # Try to download as file first, let azcopy handle the detection
-                if ! azcopy cp "$source?$AZ_SAS" "$target"; then
-                    # If failed, remove any partial target and try as directory
-                    rm -rf "$target"
-                    mkdir -p "$target"
-                    if ! azcopy cp "$source/*?$AZ_SAS" "$target" --recursive; then
-                        rm -rf "$target"
-                        >&2 echo "Unable to download path: $source"
-                        exit 1
-                    fi
-                fi
+                azcopy sync "$source?$AZ_SAS" "$target" --recursive
             }
             '''.stripIndent()
     }
@@ -58,7 +48,8 @@ class AzBashLibTest extends Specification {
     def 'should return script with config, with default azcopy opts'() {
 
         expect:
-        AzBashLib.script(new AzCopyOpts(), 10, 20, Duration.of('30s')) == '''\
+        def actual = AzBashLib.script(new AzCopyOpts(), 10, 20, Duration.of('30s'))
+        def expected = '''\
             # bash helper functions
             nxf_cp_retry() {
                 local max_attempts=20
@@ -140,22 +131,19 @@ class AzBashLibTest extends Specification {
             nxf_az_download() {
                 local source=$1
                 local target=$2
-                local basedir=$(dirname $2)
+                local basedir=$(dirname "$target")
                 mkdir -p "$basedir"
 
-                # Try to download as file first, let azcopy handle the detection
-                if ! azcopy cp "$source?$AZ_SAS" "$target"; then
-                    # If failed, remove any partial target and try as directory
-                    rm -rf "$target"
-                    mkdir -p "$target"
-                    if ! azcopy cp "$source/*?$AZ_SAS" "$target" --recursive; then
-                        rm -rf "$target"
-                        >&2 echo "Unable to download path: $source"
-                        exit 1
-                    fi
-                fi
+                azcopy sync "$source?$AZ_SAS" "$target" --recursive
             }
             '''.stripIndent()
+        
+        println "ACTUAL LENGTH: ${actual.length()}"
+        println "EXPECTED LENGTH: ${expected.length()}"
+        println "ACTUAL HEX: ${actual.getBytes().encodeHex().toString()}"
+        println "EXPECTED HEX: ${expected.getBytes().encodeHex().toString()}"
+        
+        actual == expected
     }
 
     def 'should return script with config, with custom azcopy opts'() {
@@ -243,20 +231,10 @@ class AzBashLibTest extends Specification {
             nxf_az_download() {
                 local source=$1
                 local target=$2
-                local basedir=$(dirname $2)
+                local basedir=$(dirname "$target")
                 mkdir -p "$basedir"
-            
-                # Try to download as file first, let azcopy handle the detection
-                if ! azcopy cp "$source?$AZ_SAS" "$target"; then
-                    # If failed, remove any partial target and try as directory
-                    rm -rf "$target"
-                    mkdir -p "$target"
-                    if ! azcopy cp "$source/*?$AZ_SAS" "$target" --recursive; then
-                        rm -rf "$target"
-                        >&2 echo "Unable to download path: $source"
-                        exit 1
-                    fi
-                fi
+
+                azcopy sync "$source?$AZ_SAS" "$target" --recursive
             }
             '''.stripIndent()
     }
diff --git a/plugins/nf-azure/src/test/nextflow/executor/BashWrapperBuilderWithAzTest.groovy b/plugins/nf-azure/src/test/nextflow/executor/BashWrapperBuilderWithAzTest.groovy
index fe8b11c5ed..b280fe12c4 100644
--- a/plugins/nf-azure/src/test/nextflow/executor/BashWrapperBuilderWithAzTest.groovy
+++ b/plugins/nf-azure/src/test/nextflow/executor/BashWrapperBuilderWithAzTest.groovy
@@ -77,19 +77,11 @@ class BashWrapperBuilderWithAzTest extends Specification {
             nxf_az_download() {
                 local source=$1
                 local target=$2
-                local basedir=$(dirname $2)
+                local basedir=$(dirname "$target")
                 # local ret removed
                 mkdir -p "$basedir"
-            
-                if ! azcopy cp "$source?$AZ_SAS" "$target"; then
-                    # If failed, remove any partial target and try as directory
-                    rm -rf "$target"; mkdir -p "$target"
-                    if ! azcopy cp "$source/*?$AZ_SAS" "$target" --recursive; then
-                        rm -rf "$target"
-                        >&2 echo "Unable to download path: $source"
-                        exit 1
-                    }
-                }
+
+                azcopy sync "$source?$AZ_SAS" "$target" --recursive
             }
             
             '''.stripIndent(true)
@@ -216,19 +208,11 @@ class BashWrapperBuilderWithAzTest extends Specification {
             nxf_az_download() {
                 local source=$1
                 local target=$2
-                local basedir=$(dirname $2)
+                local basedir=$(dirname "$target")
                 # local ret removed
                 mkdir -p "$basedir"
-            
-                if ! azcopy cp "$source?$AZ_SAS" "$target"; then
-                    # If failed, remove any partial target and try as directory
-                    rm -rf "$target"; mkdir -p "$target"
-                    if ! azcopy cp "$source/*?$AZ_SAS" "$target" --recursive; then
-                        rm -rf "$target"
-                        >&2 echo "Unable to download path: $source"
-                        exit 1
-                    }
-                }
+
+                azcopy sync "$source?$AZ_SAS" "$target" --recursive
             }
             
             '''.stripIndent(true)

From 5a514d7d393e0723479924e16973b97eb994d9ad Mon Sep 17 00:00:00 2001
From: adamrtalbot <12817534+adamrtalbot@users.noreply.github.com>
Date: Fri, 20 Jun 2025 16:50:01 +0100
Subject: [PATCH 6/7] fix tests

Signed-off-by: adamrtalbot <12817534+adamrtalbot@users.noreply.github.com>
---
 .../test/nextflow/executor/BashWrapperBuilderWithAzTest.groovy  | 2 --
 1 file changed, 2 deletions(-)

diff --git a/plugins/nf-azure/src/test/nextflow/executor/BashWrapperBuilderWithAzTest.groovy b/plugins/nf-azure/src/test/nextflow/executor/BashWrapperBuilderWithAzTest.groovy
index b280fe12c4..de1f43b507 100644
--- a/plugins/nf-azure/src/test/nextflow/executor/BashWrapperBuilderWithAzTest.groovy
+++ b/plugins/nf-azure/src/test/nextflow/executor/BashWrapperBuilderWithAzTest.groovy
@@ -78,7 +78,6 @@ class BashWrapperBuilderWithAzTest extends Specification {
                 local source=$1
                 local target=$2
                 local basedir=$(dirname "$target")
-                # local ret removed
                 mkdir -p "$basedir"
 
                 azcopy sync "$source?$AZ_SAS" "$target" --recursive
@@ -209,7 +208,6 @@ class BashWrapperBuilderWithAzTest extends Specification {
                 local source=$1
                 local target=$2
                 local basedir=$(dirname "$target")
-                # local ret removed
                 mkdir -p "$basedir"
 
                 azcopy sync "$source?$AZ_SAS" "$target" --recursive

From 6231d76fd633587cc3e85a44ff642cc849a2f578 Mon Sep 17 00:00:00 2001
From: adamrtalbot <12817534+adamrtalbot@users.noreply.github.com>
Date: Fri, 20 Jun 2025 18:08:37 +0100
Subject: [PATCH 7/7] Revert "Replace azcopy cp with azcopy sync in bash lib"

This reverts commit cfdf0cb236224024ea84a66db71601ab3c4cb6ea.

Signed-off-by: adamrtalbot <12817534+adamrtalbot@users.noreply.github.com>
---
 .../cloud/azure/file/AzBashLib.groovy         | 14 ++++-
 .../azure/batch/AzFileCopyStrategyTest.groovy | 48 +++++++++++++----
 .../cloud/azure/file/AzBashLibTest.groovy     | 54 +++++++++++++------
 .../BashWrapperBuilderWithAzTest.groovy       | 24 +++++++--
 4 files changed, 109 insertions(+), 31 deletions(-)

diff --git a/plugins/nf-azure/src/main/nextflow/cloud/azure/file/AzBashLib.groovy b/plugins/nf-azure/src/main/nextflow/cloud/azure/file/AzBashLib.groovy
index 31ce2774d2..1755d1f806 100644
--- a/plugins/nf-azure/src/main/nextflow/cloud/azure/file/AzBashLib.groovy
+++ b/plugins/nf-azure/src/main/nextflow/cloud/azure/file/AzBashLib.groovy
@@ -77,10 +77,20 @@ class AzBashLib extends BashFunLib<AzBashLib> {
         nxf_az_download() {
             local source=$1
             local target=$2
-            local basedir=$(dirname "$target")
+            local basedir=$(dirname $2)
             mkdir -p "$basedir"
 
-            azcopy sync "$source?$AZ_SAS" "$target" --recursive
+            # Try to download as file first, let azcopy handle the detection
+            if ! azcopy cp "$source?$AZ_SAS" "$target"; then
+                # If failed, remove any partial target and try as directory
+                rm -rf "$target"
+                mkdir -p "$target"
+                if ! azcopy cp "$source/*?$AZ_SAS" "$target" --recursive; then
+                    rm -rf "$target"
+                    >&2 echo "Unable to download path: $source"
+                    exit 1
+                fi
+            fi
         }
         '''.stripIndent()
     }
diff --git a/plugins/nf-azure/src/test/nextflow/cloud/azure/batch/AzFileCopyStrategyTest.groovy b/plugins/nf-azure/src/test/nextflow/cloud/azure/batch/AzFileCopyStrategyTest.groovy
index 7830bb30ed..fd7b77f3ec 100644
--- a/plugins/nf-azure/src/test/nextflow/cloud/azure/batch/AzFileCopyStrategyTest.groovy
+++ b/plugins/nf-azure/src/test/nextflow/cloud/azure/batch/AzFileCopyStrategyTest.groovy
@@ -178,10 +178,20 @@ class AzFileCopyStrategyTest extends Specification {
                 nxf_az_download() {
                     local source=$1
                     local target=$2
-                    local basedir=$(dirname "$target")
+                    local basedir=$(dirname $2)
                     mkdir -p "$basedir"
-
-                    azcopy sync "$source?$AZ_SAS" "$target" --recursive
+                
+                    # Try to download as file first, let azcopy handle the detection
+                    if ! azcopy cp "$source?$AZ_SAS" "$target"; then
+                        # If failed, remove any partial target and try as directory
+                        rm -rf "$target"
+                        mkdir -p "$target"
+                        if ! azcopy cp "$source/*?$AZ_SAS" "$target" --recursive; then
+                            rm -rf "$target"
+                            >&2 echo "Unable to download path: $source"
+                            exit 1
+                        fi
+                    fi
                 }
                 
                 '''.stripIndent(true)
@@ -306,10 +316,20 @@ class AzFileCopyStrategyTest extends Specification {
                 nxf_az_download() {
                     local source=$1
                     local target=$2
-                    local basedir=$(dirname "$target")
+                    local basedir=$(dirname $2)
                     mkdir -p "$basedir"
-
-                    azcopy sync "$source?$AZ_SAS" "$target" --recursive
+                
+                    # Try to download as file first, let azcopy handle the detection
+                    if ! azcopy cp "$source?$AZ_SAS" "$target"; then
+                        # If failed, remove any partial target and try as directory
+                        rm -rf "$target"
+                        mkdir -p "$target"
+                        if ! azcopy cp "$source/*?$AZ_SAS" "$target" --recursive; then
+                            rm -rf "$target"
+                            >&2 echo "Unable to download path: $source"
+                            exit 1
+                        fi
+                    fi
                 }
 
                 '''.stripIndent(true)
@@ -458,10 +478,20 @@ class AzFileCopyStrategyTest extends Specification {
                     nxf_az_download() {
                         local source=$1
                         local target=$2
-                        local basedir=$(dirname "$target")
+                        local basedir=$(dirname $2)
                         mkdir -p "$basedir"
-
-                        azcopy sync "$source?$AZ_SAS" "$target" --recursive
+                    
+                        # Try to download as file first, let azcopy handle the detection
+                        if ! azcopy cp "$source?$AZ_SAS" "$target"; then
+                            # If failed, remove any partial target and try as directory
+                            rm -rf "$target"
+                            mkdir -p "$target"
+                            if ! azcopy cp "$source/*?$AZ_SAS" "$target" --recursive; then
+                                rm -rf "$target"
+                                >&2 echo "Unable to download path: $source"
+                                exit 1
+                            fi
+                        fi
                     }
 
                     '''.stripIndent(true)
diff --git a/plugins/nf-azure/src/test/nextflow/cloud/azure/file/AzBashLibTest.groovy b/plugins/nf-azure/src/test/nextflow/cloud/azure/file/AzBashLibTest.groovy
index 00eb1c1c22..4155e77fbf 100644
--- a/plugins/nf-azure/src/test/nextflow/cloud/azure/file/AzBashLibTest.groovy
+++ b/plugins/nf-azure/src/test/nextflow/cloud/azure/file/AzBashLibTest.groovy
@@ -37,10 +37,20 @@ class AzBashLibTest extends Specification {
             nxf_az_download() {
                 local source=$1
                 local target=$2
-                local basedir=$(dirname "$target")
+                local basedir=$(dirname $2)
                 mkdir -p "$basedir"
 
-                azcopy sync "$source?$AZ_SAS" "$target" --recursive
+                # Try to download as file first, let azcopy handle the detection
+                if ! azcopy cp "$source?$AZ_SAS" "$target"; then
+                    # If failed, remove any partial target and try as directory
+                    rm -rf "$target"
+                    mkdir -p "$target"
+                    if ! azcopy cp "$source/*?$AZ_SAS" "$target" --recursive; then
+                        rm -rf "$target"
+                        >&2 echo "Unable to download path: $source"
+                        exit 1
+                    fi
+                fi
             }
             '''.stripIndent()
     }
@@ -48,8 +58,7 @@ class AzBashLibTest extends Specification {
     def 'should return script with config, with default azcopy opts'() {
 
         expect:
-        def actual = AzBashLib.script(new AzCopyOpts(), 10, 20, Duration.of('30s'))
-        def expected = '''\
+        AzBashLib.script(new AzCopyOpts(), 10, 20, Duration.of('30s')) == '''\
             # bash helper functions
             nxf_cp_retry() {
                 local max_attempts=20
@@ -131,19 +140,22 @@ class AzBashLibTest extends Specification {
             nxf_az_download() {
                 local source=$1
                 local target=$2
-                local basedir=$(dirname "$target")
+                local basedir=$(dirname $2)
                 mkdir -p "$basedir"
 
-                azcopy sync "$source?$AZ_SAS" "$target" --recursive
+                # Try to download as file first, let azcopy handle the detection
+                if ! azcopy cp "$source?$AZ_SAS" "$target"; then
+                    # If failed, remove any partial target and try as directory
+                    rm -rf "$target"
+                    mkdir -p "$target"
+                    if ! azcopy cp "$source/*?$AZ_SAS" "$target" --recursive; then
+                        rm -rf "$target"
+                        >&2 echo "Unable to download path: $source"
+                        exit 1
+                    fi
+                fi
             }
             '''.stripIndent()
-        
-        println "ACTUAL LENGTH: ${actual.length()}"
-        println "EXPECTED LENGTH: ${expected.length()}"
-        println "ACTUAL HEX: ${actual.getBytes().encodeHex().toString()}"
-        println "EXPECTED HEX: ${expected.getBytes().encodeHex().toString()}"
-        
-        actual == expected
     }
 
     def 'should return script with config, with custom azcopy opts'() {
@@ -231,10 +243,20 @@ class AzBashLibTest extends Specification {
             nxf_az_download() {
                 local source=$1
                 local target=$2
-                local basedir=$(dirname "$target")
+                local basedir=$(dirname $2)
                 mkdir -p "$basedir"
-
-                azcopy sync "$source?$AZ_SAS" "$target" --recursive
+            
+                # Try to download as file first, let azcopy handle the detection
+                if ! azcopy cp "$source?$AZ_SAS" "$target"; then
+                    # If failed, remove any partial target and try as directory
+                    rm -rf "$target"
+                    mkdir -p "$target"
+                    if ! azcopy cp "$source/*?$AZ_SAS" "$target" --recursive; then
+                        rm -rf "$target"
+                        >&2 echo "Unable to download path: $source"
+                        exit 1
+                    fi
+                fi
             }
             '''.stripIndent()
     }
diff --git a/plugins/nf-azure/src/test/nextflow/executor/BashWrapperBuilderWithAzTest.groovy b/plugins/nf-azure/src/test/nextflow/executor/BashWrapperBuilderWithAzTest.groovy
index de1f43b507..d6a380fb60 100644
--- a/plugins/nf-azure/src/test/nextflow/executor/BashWrapperBuilderWithAzTest.groovy
+++ b/plugins/nf-azure/src/test/nextflow/executor/BashWrapperBuilderWithAzTest.groovy
@@ -79,8 +79,16 @@ class BashWrapperBuilderWithAzTest extends Specification {
                 local target=$2
                 local basedir=$(dirname "$target")
                 mkdir -p "$basedir"
-
-                azcopy sync "$source?$AZ_SAS" "$target" --recursive
+            
+                if ! azcopy cp "$source?$AZ_SAS" "$target"; then
+                    # If failed, remove any partial target and try as directory
+                    rm -rf "$target"; mkdir -p "$target"
+                    if ! azcopy cp "$source/*?$AZ_SAS" "$target" --recursive; then
+                        rm -rf "$target"
+                        >&2 echo "Unable to download path: $source"
+                        exit 1
+                    }
+                }
             }
             
             '''.stripIndent(true)
@@ -209,8 +217,16 @@ class BashWrapperBuilderWithAzTest extends Specification {
                 local target=$2
                 local basedir=$(dirname "$target")
                 mkdir -p "$basedir"
-
-                azcopy sync "$source?$AZ_SAS" "$target" --recursive
+            
+                if ! azcopy cp "$source?$AZ_SAS" "$target"; then
+                    # If failed, remove any partial target and try as directory
+                    rm -rf "$target"; mkdir -p "$target"
+                    if ! azcopy cp "$source/*?$AZ_SAS" "$target" --recursive; then
+                        rm -rf "$target"
+                        >&2 echo "Unable to download path: $source"
+                        exit 1
+                    }
+                }
             }
             
             '''.stripIndent(true)