Skip to content

Commit 0053f94

Browse files
adamrtalbotadamrtalbot
committed
Handle azcopy failure better
Signed-off-by: adamrtalbot <12817534+adamrtalbot@users.noreply.github.com>
1 parent 8ffe60b commit 0053f94

File tree

3 files changed

+59
-54
lines changed

3 files changed

+59
-54
lines changed

‎plugins/nf-azure/src/test/nextflow/cloud/azure/batch/AzFileCopyStrategyTest.groovy

Lines changed: 18 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -316,18 +316,19 @@ class AzFileCopyStrategyTest extends Specification {
316316
local source=$1
317317
local target=$2
318318
local basedir=$(dirname $2)
319-
local ret
320319
mkdir -p "$basedir"
321320
322-
ret=$(azcopy cp "$source?$AZ_SAS" "$target" 2>&1) || {
323-
## if fails check if it was trying to download a directory
324-
mkdir -p $target
325-
azcopy cp "$source/*?$AZ_SAS" "$target" --recursive >/dev/null || {
326-
rm -rf $target
321+
# Try to download as file first, let azcopy handle the detection
322+
if ! azcopy cp "$source?$AZ_SAS" "$target"; then
323+
# If failed, remove any partial target and try as directory
324+
rm -rf "$target"
325+
mkdir -p "$target"
326+
if ! azcopy cp "$source/*?$AZ_SAS" "$target" --recursive; then
327+
rm -rf "$target"
327328
>&2 echo "Unable to download path: $source"
328329
exit 1
329-
}
330-
}
330+
fi
331+
fi
331332
}
332333
333334
'''.stripIndent(true)
@@ -477,18 +478,19 @@ class AzFileCopyStrategyTest extends Specification {
477478
local source=$1
478479
local target=$2
479480
local basedir=$(dirname $2)
480-
local ret
481481
mkdir -p "$basedir"
482482
483-
ret=$(azcopy cp "$source?$AZ_SAS" "$target" 2>&1) || {
484-
## if fails check if it was trying to download a directory
485-
mkdir -p $target
486-
azcopy cp "$source/*?$AZ_SAS" "$target" --recursive >/dev/null || {
487-
rm -rf $target
483+
# Try to download as file first, let azcopy handle the detection
484+
if ! azcopy cp "$source?$AZ_SAS" "$target"; then
485+
# If failed, remove any partial target and try as directory
486+
rm -rf "$target"
487+
mkdir -p "$target"
488+
if ! azcopy cp "$source/*?$AZ_SAS" "$target" --recursive; then
489+
rm -rf "$target"
488490
>&2 echo "Unable to download path: $source"
489491
exit 1
490-
}
491-
}
492+
fi
493+
fi
492494
}
493495
494496
'''.stripIndent(true)

‎plugins/nf-azure/src/test/nextflow/cloud/azure/file/AzBashLibTest.groovy

Lines changed: 29 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -38,18 +38,19 @@ class AzBashLibTest extends Specification {
3838
local source=$1
3939
local target=$2
4040
local basedir=$(dirname $2)
41-
local ret
4241
mkdir -p "$basedir"
4342
44-
ret=$(azcopy cp "$source?$AZ_SAS" "$target" 2>&1) || {
45-
## if fails check if it was trying to download a directory
46-
mkdir -p $target
47-
azcopy cp "$source/*?$AZ_SAS" "$target" --recursive >/dev/null || {
48-
rm -rf $target
43+
# Try to download as file first, let azcopy handle the detection
44+
if ! azcopy cp "$source?$AZ_SAS" "$target"; then
45+
# If failed, remove any partial target and try as directory
46+
rm -rf "$target"
47+
mkdir -p "$target"
48+
if ! azcopy cp "$source/*?$AZ_SAS" "$target" --recursive; then
49+
rm -rf "$target"
4950
>&2 echo "Unable to download path: $source"
5051
exit 1
51-
}
52-
}
52+
fi
53+
fi
5354
}
5455
'''.stripIndent()
5556
}
@@ -136,22 +137,23 @@ class AzBashLibTest extends Specification {
136137
fi
137138
}
138139
139-
nxf_az_download() {
140+
nxf_az_download() {
140141
local source=$1
141142
local target=$2
142143
local basedir=$(dirname $2)
143-
local ret
144144
mkdir -p "$basedir"
145-
146-
ret=$(azcopy cp "$source?$AZ_SAS" "$target" 2>&1) || {
147-
## if fails check if it was trying to download a directory
148-
mkdir -p $target
149-
azcopy cp "$source/*?$AZ_SAS" "$target" --recursive >/dev/null || {
150-
rm -rf $target
145+
146+
# Try to download as file first, let azcopy handle the detection
147+
if ! azcopy cp "$source?$AZ_SAS" "$target"; then
148+
# If failed, remove any partial target and try as directory
149+
rm -rf "$target"
150+
mkdir -p "$target"
151+
if ! azcopy cp "$source/*?$AZ_SAS" "$target" --recursive; then
152+
rm -rf "$target"
151153
>&2 echo "Unable to download path: $source"
152154
exit 1
153-
}
154-
}
155+
fi
156+
fi
155157
}
156158
'''.stripIndent()
157159
}
@@ -242,18 +244,19 @@ class AzBashLibTest extends Specification {
242244
local source=$1
243245
local target=$2
244246
local basedir=$(dirname $2)
245-
local ret
246247
mkdir -p "$basedir"
247248
248-
ret=$(azcopy cp "$source?$AZ_SAS" "$target" 2>&1) || {
249-
## if fails check if it was trying to download a directory
250-
mkdir -p $target
251-
azcopy cp "$source/*?$AZ_SAS" "$target" --recursive >/dev/null || {
252-
rm -rf $target
249+
# Try to download as file first, let azcopy handle the detection
250+
if ! azcopy cp "$source?$AZ_SAS" "$target"; then
251+
# If failed, remove any partial target and try as directory
252+
rm -rf "$target"
253+
mkdir -p "$target"
254+
if ! azcopy cp "$source/*?$AZ_SAS" "$target" --recursive; then
255+
rm -rf "$target"
253256
>&2 echo "Unable to download path: $source"
254257
exit 1
255-
}
256-
}
258+
fi
259+
fi
257260
}
258261
'''.stripIndent()
259262
}

‎plugins/nf-azure/src/test/nextflow/executor/BashWrapperBuilderWithAzTest.groovy

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -78,14 +78,14 @@ class BashWrapperBuilderWithAzTest extends Specification {
7878
local source=$1
7979
local target=$2
8080
local basedir=$(dirname $2)
81-
local ret
81+
# local ret removed
8282
mkdir -p "$basedir"
8383
84-
ret=$(azcopy cp "$source?$AZ_SAS" "$target" 2>&1) || {
85-
## if fails check if it was trying to download a directory
86-
mkdir -p $target
87-
azcopy cp "$source/*?$AZ_SAS" "$target" --recursive >/dev/null || {
88-
rm -rf $target
84+
if ! azcopy cp "$source?$AZ_SAS" "$target"; then
85+
# If failed, remove any partial target and try as directory
86+
rm -rf "$target"; mkdir -p "$target"
87+
if ! azcopy cp "$source/*?$AZ_SAS" "$target" --recursive; then
88+
rm -rf "$target"
8989
>&2 echo "Unable to download path: $source"
9090
exit 1
9191
}
@@ -217,14 +217,14 @@ class BashWrapperBuilderWithAzTest extends Specification {
217217
local source=$1
218218
local target=$2
219219
local basedir=$(dirname $2)
220-
local ret
220+
# local ret removed
221221
mkdir -p "$basedir"
222222
223-
ret=$(azcopy cp "$source?$AZ_SAS" "$target" 2>&1) || {
224-
## if fails check if it was trying to download a directory
225-
mkdir -p $target
226-
azcopy cp "$source/*?$AZ_SAS" "$target" --recursive >/dev/null || {
227-
rm -rf $target
223+
if ! azcopy cp "$source?$AZ_SAS" "$target"; then
224+
# If failed, remove any partial target and try as directory
225+
rm -rf "$target"; mkdir -p "$target"
226+
if ! azcopy cp "$source/*?$AZ_SAS" "$target" --recursive; then
227+
rm -rf "$target"
228228
>&2 echo "Unable to download path: $source"
229229
exit 1
230230
}

0 commit comments

Comments
 (0)