Skip to content

Commit cfdf0cb

Browse files
adamrtalbotadamrtalbot
committed
Replace azcopy cp with azcopy sync in bash lib
This seems to work with both files and directories with no issue. I tested it with: ```bash make pack && ./build/releases/nextflow-25.05.0-edge-dist run nf-core/rnaseq -r 3.19.0 -profile test --outdir results -c azure.config -w az://work/ BUILD_PACK=1 ./gradlew pack ``` And it worked fine? Signed-off-by: adamrtalbot <12817534+adamrtalbot@users.noreply.github.com>
1 parent 14a2491 commit cfdf0cb

File tree

4 files changed

+33
-111
lines changed

4 files changed

+33
-111
lines changed

‎plugins/nf-azure/src/main/nextflow/cloud/azure/file/AzBashLib.groovy

Lines changed: 2 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -77,20 +77,10 @@ class AzBashLib extends BashFunLib<AzBashLib> {
7777
nxf_az_download() {
7878
local source=$1
7979
local target=$2
80-
local basedir=$(dirname $2)
80+
local basedir=$(dirname "$target")
8181
mkdir -p "$basedir"
8282
83-
# Try to download as file first, let azcopy handle the detection
84-
if ! azcopy cp "$source?$AZ_SAS" "$target"; then
85-
# If failed, remove any partial target and try as directory
86-
rm -rf "$target"
87-
mkdir -p "$target"
88-
if ! azcopy cp "$source/*?$AZ_SAS" "$target" --recursive; then
89-
rm -rf "$target"
90-
>&2 echo "Unable to download path: $source"
91-
exit 1
92-
fi
93-
fi
83+
azcopy sync "$source?$AZ_SAS" "$target" --recursive
9484
}
9585
'''.stripIndent()
9686
}

‎plugins/nf-azure/src/test/nextflow/cloud/azure/batch/AzFileCopyStrategyTest.groovy

Lines changed: 9 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -178,20 +178,10 @@ class AzFileCopyStrategyTest extends Specification {
178178
nxf_az_download() {
179179
local source=$1
180180
local target=$2
181-
local basedir=$(dirname $2)
181+
local basedir=$(dirname "$target")
182182
mkdir -p "$basedir"
183-
184-
# Try to download as file first, let azcopy handle the detection
185-
if ! azcopy cp "$source?$AZ_SAS" "$target"; then
186-
# If failed, remove any partial target and try as directory
187-
rm -rf "$target"
188-
mkdir -p "$target"
189-
if ! azcopy cp "$source/*?$AZ_SAS" "$target" --recursive; then
190-
rm -rf "$target"
191-
>&2 echo "Unable to download path: $source"
192-
exit 1
193-
fi
194-
fi
183+
184+
azcopy sync "$source?$AZ_SAS" "$target" --recursive
195185
}
196186
197187
'''.stripIndent(true)
@@ -316,20 +306,10 @@ class AzFileCopyStrategyTest extends Specification {
316306
nxf_az_download() {
317307
local source=$1
318308
local target=$2
319-
local basedir=$(dirname $2)
309+
local basedir=$(dirname "$target")
320310
mkdir -p "$basedir"
321-
322-
# Try to download as file first, let azcopy handle the detection
323-
if ! azcopy cp "$source?$AZ_SAS" "$target"; then
324-
# If failed, remove any partial target and try as directory
325-
rm -rf "$target"
326-
mkdir -p "$target"
327-
if ! azcopy cp "$source/*?$AZ_SAS" "$target" --recursive; then
328-
rm -rf "$target"
329-
>&2 echo "Unable to download path: $source"
330-
exit 1
331-
fi
332-
fi
311+
312+
azcopy sync "$source?$AZ_SAS" "$target" --recursive
333313
}
334314
335315
'''.stripIndent(true)
@@ -478,20 +458,10 @@ class AzFileCopyStrategyTest extends Specification {
478458
nxf_az_download() {
479459
local source=$1
480460
local target=$2
481-
local basedir=$(dirname $2)
461+
local basedir=$(dirname "$target")
482462
mkdir -p "$basedir"
483-
484-
# Try to download as file first, let azcopy handle the detection
485-
if ! azcopy cp "$source?$AZ_SAS" "$target"; then
486-
# If failed, remove any partial target and try as directory
487-
rm -rf "$target"
488-
mkdir -p "$target"
489-
if ! azcopy cp "$source/*?$AZ_SAS" "$target" --recursive; then
490-
rm -rf "$target"
491-
>&2 echo "Unable to download path: $source"
492-
exit 1
493-
fi
494-
fi
463+
464+
azcopy sync "$source?$AZ_SAS" "$target" --recursive
495465
}
496466
497467
'''.stripIndent(true)

‎plugins/nf-azure/src/test/nextflow/cloud/azure/file/AzBashLibTest.groovy

Lines changed: 16 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -37,28 +37,19 @@ class AzBashLibTest extends Specification {
3737
nxf_az_download() {
3838
local source=$1
3939
local target=$2
40-
local basedir=$(dirname $2)
40+
local basedir=$(dirname "$target")
4141
mkdir -p "$basedir"
4242
43-
# Try to download as file first, let azcopy handle the detection
44-
if ! azcopy cp "$source?$AZ_SAS" "$target"; then
45-
# If failed, remove any partial target and try as directory
46-
rm -rf "$target"
47-
mkdir -p "$target"
48-
if ! azcopy cp "$source/*?$AZ_SAS" "$target" --recursive; then
49-
rm -rf "$target"
50-
>&2 echo "Unable to download path: $source"
51-
exit 1
52-
fi
53-
fi
43+
azcopy sync "$source?$AZ_SAS" "$target" --recursive
5444
}
5545
'''.stripIndent()
5646
}
5747

5848
def 'should return script with config, with default azcopy opts'() {
5949

6050
expect:
61-
AzBashLib.script(new AzCopyOpts(), 10, 20, Duration.of('30s')) == '''\
51+
def actual = AzBashLib.script(new AzCopyOpts(), 10, 20, Duration.of('30s'))
52+
def expected = '''\
6253
# bash helper functions
6354
nxf_cp_retry() {
6455
local max_attempts=20
@@ -140,22 +131,19 @@ class AzBashLibTest extends Specification {
140131
nxf_az_download() {
141132
local source=$1
142133
local target=$2
143-
local basedir=$(dirname $2)
134+
local basedir=$(dirname "$target")
144135
mkdir -p "$basedir"
145136
146-
# Try to download as file first, let azcopy handle the detection
147-
if ! azcopy cp "$source?$AZ_SAS" "$target"; then
148-
# If failed, remove any partial target and try as directory
149-
rm -rf "$target"
150-
mkdir -p "$target"
151-
if ! azcopy cp "$source/*?$AZ_SAS" "$target" --recursive; then
152-
rm -rf "$target"
153-
>&2 echo "Unable to download path: $source"
154-
exit 1
155-
fi
156-
fi
137+
azcopy sync "$source?$AZ_SAS" "$target" --recursive
157138
}
158139
'''.stripIndent()
140+
141+
println "ACTUAL LENGTH: ${actual.length()}"
142+
println "EXPECTED LENGTH: ${expected.length()}"
143+
println "ACTUAL HEX: ${actual.getBytes().encodeHex().toString()}"
144+
println "EXPECTED HEX: ${expected.getBytes().encodeHex().toString()}"
145+
146+
actual == expected
159147
}
160148

161149
def 'should return script with config, with custom azcopy opts'() {
@@ -243,20 +231,10 @@ class AzBashLibTest extends Specification {
243231
nxf_az_download() {
244232
local source=$1
245233
local target=$2
246-
local basedir=$(dirname $2)
234+
local basedir=$(dirname "$target")
247235
mkdir -p "$basedir"
248-
249-
# Try to download as file first, let azcopy handle the detection
250-
if ! azcopy cp "$source?$AZ_SAS" "$target"; then
251-
# If failed, remove any partial target and try as directory
252-
rm -rf "$target"
253-
mkdir -p "$target"
254-
if ! azcopy cp "$source/*?$AZ_SAS" "$target" --recursive; then
255-
rm -rf "$target"
256-
>&2 echo "Unable to download path: $source"
257-
exit 1
258-
fi
259-
fi
236+
237+
azcopy sync "$source?$AZ_SAS" "$target" --recursive
260238
}
261239
'''.stripIndent()
262240
}

‎plugins/nf-azure/src/test/nextflow/executor/BashWrapperBuilderWithAzTest.groovy

Lines changed: 6 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -77,19 +77,11 @@ class BashWrapperBuilderWithAzTest extends Specification {
7777
nxf_az_download() {
7878
local source=$1
7979
local target=$2
80-
local basedir=$(dirname $2)
80+
local basedir=$(dirname "$target")
8181
# local ret removed
8282
mkdir -p "$basedir"
83-
84-
if ! azcopy cp "$source?$AZ_SAS" "$target"; then
85-
# If failed, remove any partial target and try as directory
86-
rm -rf "$target"; mkdir -p "$target"
87-
if ! azcopy cp "$source/*?$AZ_SAS" "$target" --recursive; then
88-
rm -rf "$target"
89-
>&2 echo "Unable to download path: $source"
90-
exit 1
91-
}
92-
}
83+
84+
azcopy sync "$source?$AZ_SAS" "$target" --recursive
9385
}
9486
9587
'''.stripIndent(true)
@@ -216,19 +208,11 @@ class BashWrapperBuilderWithAzTest extends Specification {
216208
nxf_az_download() {
217209
local source=$1
218210
local target=$2
219-
local basedir=$(dirname $2)
211+
local basedir=$(dirname "$target")
220212
# local ret removed
221213
mkdir -p "$basedir"
222-
223-
if ! azcopy cp "$source?$AZ_SAS" "$target"; then
224-
# If failed, remove any partial target and try as directory
225-
rm -rf "$target"; mkdir -p "$target"
226-
if ! azcopy cp "$source/*?$AZ_SAS" "$target" --recursive; then
227-
rm -rf "$target"
228-
>&2 echo "Unable to download path: $source"
229-
exit 1
230-
}
231-
}
214+
215+
azcopy sync "$source?$AZ_SAS" "$target" --recursive
232216
}
233217
234218
'''.stripIndent(true)

0 commit comments

Comments
 (0)