Skip to content

Commit 6231d76

Browse files
adamrtalbotadamrtalbot
committed
Revert "Replace azcopy cp with azcopy sync in bash lib"
This reverts commit cfdf0cb. Signed-off-by: adamrtalbot <12817534+adamrtalbot@users.noreply.github.com>
1 parent 5a514d7 commit 6231d76

File tree

4 files changed

+109
-31
lines changed

4 files changed

+109
-31
lines changed

‎plugins/nf-azure/src/main/nextflow/cloud/azure/file/AzBashLib.groovy

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -77,10 +77,20 @@ class AzBashLib extends BashFunLib<AzBashLib> {
7777
nxf_az_download() {
7878
local source=$1
7979
local target=$2
80-
local basedir=$(dirname "$target")
80+
local basedir=$(dirname $2)
8181
mkdir -p "$basedir"
8282
83-
azcopy sync "$source?$AZ_SAS" "$target" --recursive
83+
# Try to download as file first, let azcopy handle the detection
84+
if ! azcopy cp "$source?$AZ_SAS" "$target"; then
85+
# If failed, remove any partial target and try as directory
86+
rm -rf "$target"
87+
mkdir -p "$target"
88+
if ! azcopy cp "$source/*?$AZ_SAS" "$target" --recursive; then
89+
rm -rf "$target"
90+
>&2 echo "Unable to download path: $source"
91+
exit 1
92+
fi
93+
fi
8494
}
8595
'''.stripIndent()
8696
}

‎plugins/nf-azure/src/test/nextflow/cloud/azure/batch/AzFileCopyStrategyTest.groovy

Lines changed: 39 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -178,10 +178,20 @@ class AzFileCopyStrategyTest extends Specification {
178178
nxf_az_download() {
179179
local source=$1
180180
local target=$2
181-
local basedir=$(dirname "$target")
181+
local basedir=$(dirname $2)
182182
mkdir -p "$basedir"
183-
184-
azcopy sync "$source?$AZ_SAS" "$target" --recursive
183+
184+
# Try to download as file first, let azcopy handle the detection
185+
if ! azcopy cp "$source?$AZ_SAS" "$target"; then
186+
# If failed, remove any partial target and try as directory
187+
rm -rf "$target"
188+
mkdir -p "$target"
189+
if ! azcopy cp "$source/*?$AZ_SAS" "$target" --recursive; then
190+
rm -rf "$target"
191+
>&2 echo "Unable to download path: $source"
192+
exit 1
193+
fi
194+
fi
185195
}
186196
187197
'''.stripIndent(true)
@@ -306,10 +316,20 @@ class AzFileCopyStrategyTest extends Specification {
306316
nxf_az_download() {
307317
local source=$1
308318
local target=$2
309-
local basedir=$(dirname "$target")
319+
local basedir=$(dirname $2)
310320
mkdir -p "$basedir"
311-
312-
azcopy sync "$source?$AZ_SAS" "$target" --recursive
321+
322+
# Try to download as file first, let azcopy handle the detection
323+
if ! azcopy cp "$source?$AZ_SAS" "$target"; then
324+
# If failed, remove any partial target and try as directory
325+
rm -rf "$target"
326+
mkdir -p "$target"
327+
if ! azcopy cp "$source/*?$AZ_SAS" "$target" --recursive; then
328+
rm -rf "$target"
329+
>&2 echo "Unable to download path: $source"
330+
exit 1
331+
fi
332+
fi
313333
}
314334
315335
'''.stripIndent(true)
@@ -458,10 +478,20 @@ class AzFileCopyStrategyTest extends Specification {
458478
nxf_az_download() {
459479
local source=$1
460480
local target=$2
461-
local basedir=$(dirname "$target")
481+
local basedir=$(dirname $2)
462482
mkdir -p "$basedir"
463-
464-
azcopy sync "$source?$AZ_SAS" "$target" --recursive
483+
484+
# Try to download as file first, let azcopy handle the detection
485+
if ! azcopy cp "$source?$AZ_SAS" "$target"; then
486+
# If failed, remove any partial target and try as directory
487+
rm -rf "$target"
488+
mkdir -p "$target"
489+
if ! azcopy cp "$source/*?$AZ_SAS" "$target" --recursive; then
490+
rm -rf "$target"
491+
>&2 echo "Unable to download path: $source"
492+
exit 1
493+
fi
494+
fi
465495
}
466496
467497
'''.stripIndent(true)

‎plugins/nf-azure/src/test/nextflow/cloud/azure/file/AzBashLibTest.groovy

Lines changed: 38 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -37,19 +37,28 @@ class AzBashLibTest extends Specification {
3737
nxf_az_download() {
3838
local source=$1
3939
local target=$2
40-
local basedir=$(dirname "$target")
40+
local basedir=$(dirname $2)
4141
mkdir -p "$basedir"
4242
43-
azcopy sync "$source?$AZ_SAS" "$target" --recursive
43+
# Try to download as file first, let azcopy handle the detection
44+
if ! azcopy cp "$source?$AZ_SAS" "$target"; then
45+
# If failed, remove any partial target and try as directory
46+
rm -rf "$target"
47+
mkdir -p "$target"
48+
if ! azcopy cp "$source/*?$AZ_SAS" "$target" --recursive; then
49+
rm -rf "$target"
50+
>&2 echo "Unable to download path: $source"
51+
exit 1
52+
fi
53+
fi
4454
}
4555
'''.stripIndent()
4656
}
4757

4858
def 'should return script with config, with default azcopy opts'() {
4959

5060
expect:
51-
def actual = AzBashLib.script(new AzCopyOpts(), 10, 20, Duration.of('30s'))
52-
def expected = '''\
61+
AzBashLib.script(new AzCopyOpts(), 10, 20, Duration.of('30s')) == '''\
5362
# bash helper functions
5463
nxf_cp_retry() {
5564
local max_attempts=20
@@ -131,19 +140,22 @@ class AzBashLibTest extends Specification {
131140
nxf_az_download() {
132141
local source=$1
133142
local target=$2
134-
local basedir=$(dirname "$target")
143+
local basedir=$(dirname $2)
135144
mkdir -p "$basedir"
136145
137-
azcopy sync "$source?$AZ_SAS" "$target" --recursive
146+
# Try to download as file first, let azcopy handle the detection
147+
if ! azcopy cp "$source?$AZ_SAS" "$target"; then
148+
# If failed, remove any partial target and try as directory
149+
rm -rf "$target"
150+
mkdir -p "$target"
151+
if ! azcopy cp "$source/*?$AZ_SAS" "$target" --recursive; then
152+
rm -rf "$target"
153+
>&2 echo "Unable to download path: $source"
154+
exit 1
155+
fi
156+
fi
138157
}
139158
'''.stripIndent()
140-
141-
println "ACTUAL LENGTH: ${actual.length()}"
142-
println "EXPECTED LENGTH: ${expected.length()}"
143-
println "ACTUAL HEX: ${actual.getBytes().encodeHex().toString()}"
144-
println "EXPECTED HEX: ${expected.getBytes().encodeHex().toString()}"
145-
146-
actual == expected
147159
}
148160

149161
def 'should return script with config, with custom azcopy opts'() {
@@ -231,10 +243,20 @@ class AzBashLibTest extends Specification {
231243
nxf_az_download() {
232244
local source=$1
233245
local target=$2
234-
local basedir=$(dirname "$target")
246+
local basedir=$(dirname $2)
235247
mkdir -p "$basedir"
236-
237-
azcopy sync "$source?$AZ_SAS" "$target" --recursive
248+
249+
# Try to download as file first, let azcopy handle the detection
250+
if ! azcopy cp "$source?$AZ_SAS" "$target"; then
251+
# If failed, remove any partial target and try as directory
252+
rm -rf "$target"
253+
mkdir -p "$target"
254+
if ! azcopy cp "$source/*?$AZ_SAS" "$target" --recursive; then
255+
rm -rf "$target"
256+
>&2 echo "Unable to download path: $source"
257+
exit 1
258+
fi
259+
fi
238260
}
239261
'''.stripIndent()
240262
}

‎plugins/nf-azure/src/test/nextflow/executor/BashWrapperBuilderWithAzTest.groovy

Lines changed: 20 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -79,8 +79,16 @@ class BashWrapperBuilderWithAzTest extends Specification {
7979
local target=$2
8080
local basedir=$(dirname "$target")
8181
mkdir -p "$basedir"
82-
83-
azcopy sync "$source?$AZ_SAS" "$target" --recursive
82+
83+
if ! azcopy cp "$source?$AZ_SAS" "$target"; then
84+
# If failed, remove any partial target and try as directory
85+
rm -rf "$target"; mkdir -p "$target"
86+
if ! azcopy cp "$source/*?$AZ_SAS" "$target" --recursive; then
87+
rm -rf "$target"
88+
>&2 echo "Unable to download path: $source"
89+
exit 1
90+
}
91+
}
8492
}
8593
8694
'''.stripIndent(true)
@@ -209,8 +217,16 @@ class BashWrapperBuilderWithAzTest extends Specification {
209217
local target=$2
210218
local basedir=$(dirname "$target")
211219
mkdir -p "$basedir"
212-
213-
azcopy sync "$source?$AZ_SAS" "$target" --recursive
220+
221+
if ! azcopy cp "$source?$AZ_SAS" "$target"; then
222+
# If failed, remove any partial target and try as directory
223+
rm -rf "$target"; mkdir -p "$target"
224+
if ! azcopy cp "$source/*?$AZ_SAS" "$target" --recursive; then
225+
rm -rf "$target"
226+
>&2 echo "Unable to download path: $source"
227+
exit 1
228+
}
229+
}
214230
}
215231
216232
'''.stripIndent(true)

0 commit comments

Comments
 (0)