Implement retry strategy for Git repository provider (#6195)

Signed-off-by: Paolo Di Tommaso <paolo.ditommaso@gmail.com>

Author: pditommaso

docs/reference/config.md

@@ -1185,10 +1185,26 @@ Read the {ref}`sharing-page` page to learn how to publish your pipeline to GitHu
 
 ## `nextflow`
 
-:::{deprecated} 24.10.0
-The `nextflow.publish` scope has been renamed to `workflow.output`. See {ref}`config-workflow` for more information.
+:::{versionchanged} 24.10.0
+The `nextflow.publish.retryPolicy` settings were moved to `workflow.output.retryPolicy`.
 :::
 
+:::{versionchanged} 25.06.0-edge
+The `workflow.output.retryPolicy` settings were moved to `nextflow.retryPolicy`.
+:::
+
+`retryPolicy.delay`
+: Delay used for retryable operations (default: `350ms`).
+
+`retryPolicy.jitter`
+: Jitter value used for retryable operations (default: `0.25`).
+
+`retryPolicy.maxAttempts`
+: Max attempts used for retryable operations (default: `5`).
+
+`retryPolicy.maxDelay`
+: Max delay used for retryable operations (default: `90s`).
+
 (config-notification)=
 
 ## `notification`
@@ -1644,18 +1660,6 @@ The `workflow` scope provides workflow execution options.
   `'standard'`
   : Overwrite existing files when the file size or last modified timestamp is different.
 
-`workflow.output.retryPolicy.delay`
-: Delay when retrying a failed publish operation (default: `350ms`).
-
-`workflow.output.retryPolicy.jitter`
-: Jitter value when retrying a failed publish operation (default: `0.25`).
-
-`workflow.output.retryPolicy.maxAttempt`
-: Max attempts when retrying a failed publish operation (default: `5`).
-
-`workflow.output.retryPolicy.maxDelay`
-: Max delay when retrying a failed publish operation (default: `90s`).
-
 `workflow.output.storageClass`
 : *Currently only supported for S3.*
 : Specify the storage class for published files.

docs/reference/env-vars.md

@@ -166,6 +166,26 @@ The following environment variables control the configuration of the Nextflow ru
   :::
 : Defines the default behavior of `publishDir.failOnError` setting. See {ref}`publishDir<process-publishdir>` directive for more information.
 
+`NXF_RETRY_POLICY_DELAY`
+: :::{versionadded} 25.06.0-edge
+  :::
+: Delay used for HTTP retryable operations (default: `350ms`).
+
+`NXF_RETRY_POLICY_JITTER`
+: :::{versionadded} 25.06.0-edge
+  :::
+: Jitter value used for HTTP retryable operations (default: `0.25`).
+
+`NXF_RETRY_POLICY_MAX_ATTEMPTS`
+: :::{versionadded} 25.06.0-edge
+  :::
+: Max number of attempts used for HTTP retryable operations (default: `5`).
+
+`NXF_RETRY_POLICY_MAX_DELAY`
+: :::{versionadded} 25.06.0-edge
+  :::
+: Max delay used for HTTP retryable operations (default: `90s`).
+
 `NXF_SCM_FILE`
 : :::{versionadded} 20.10.0
   :::

modules/nextflow/src/main/groovy/nextflow/processor/PublishDir.groovy

@@ -52,6 +52,7 @@ import nextflow.file.TagAwareFile
 import nextflow.trace.event.FilePublishEvent
 import nextflow.util.HashBuilder
 import nextflow.util.PathTrie
+import nextflow.util.RetryConfig
 /**
  * Implements the {@code publishDir} directory. It create links or copies the output
  * files of a given task to a user specified directory.
@@ -127,7 +128,7 @@ class PublishDir {
      */
     private String storageClass
 
-    private PublishRetryConfig retryConfig
+    private RetryConfig retryConfig
 
     private PathMatcher matcher
 
@@ -231,19 +232,10 @@ class PublishDir {
         return result
     }
 
-    protected Map getRetryOpts() {
-        def result = session.config.navigate('nextflow.publish.retryPolicy') as Map
-        if( result != null )
-            log.warn 'The `nextflow.publish` config scope has been renamed to `workflow.output`'
-        else
-            result = session.config.navigate('workflow.output.retryPolicy') as Map ?: Collections.emptyMap()
-        return result
-    }
-
     protected void apply0(Set<Path> files) {
         assert path
         // setup the retry policy config to be used
-        this.retryConfig = new PublishRetryConfig(getRetryOpts())
+        this.retryConfig = RetryConfig.config(session)
 
         createPublishDir()
         validatePublishMode()

modules/nextflow/src/main/groovy/nextflow/processor/PublishRetryConfig.groovy

@@ -161,14 +161,14 @@ final class AzureRepositoryProvider extends RepositoryProvider {
      * Check for response error status. Throws a {@link nextflow.exception.AbortOperationException} exception
      * when a 401 or 403 error status is returned.
      *
-     * @param connection A {@link HttpURLConnection} connection instance
+     * @param response A {@link HttpURLConnection} connection instance
      */
-    protected checkResponse( HttpResponse<String> connection ) {
-        this.continuationToken = connection
+    protected checkResponse( HttpResponse<String> response) {
+        this.continuationToken = response
                 .headers()
                 .firstValue("x-ms-continuationtoken")
                 .orElse(null)
-        super.checkResponse(connection)
+        super.checkResponse(response)
     }
 
     /** {@inheritDoc} */