Merge branch 'master' into azure_batch_no_container_spec

Author: adamrtalbot

.github/workflows/seqeradocs-changelog.yml

@@ -0,0 +1,62 @@
+name: Push release change log to Seqera Docs
+
+on:
+  release:
+    types: [published]
+  workflow_dispatch:
+    inputs:
+      release_name:
+        description: "Release version (e.g. 1.0.0)"
+        required: true
+      release_body:
+        description: "Release changelog content"
+        required: true
+
+jobs:
+  update-docs:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Clone seqeralabs/docs
+        run: |
+          git clone https://github.com/seqeralabs/docs.git seqeralabs-docs
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Create changelog file
+        run: |
+          mkdir -p seqeralabs-docs/changelog/nextflow
+          cat << EOF > seqeralabs-docs/changelog/nextflow/${{ github.event.release.name || inputs.release_name }}.mdx
+          ---
+          title: Nextflow ${{ github.event.release.name || inputs.release_name }}
+          date: $(date +%Y-%m-%d)
+          tags: [nextflow]
+          ---
+
+          ${{ github.event.release.body || inputs.release_body }}
+          EOF
+
+      - uses: actions/create-github-app-token@v1
+        id: generate-token
+        with:
+          app-id: ${{ secrets.DOCS_BOT_APP_ID }}
+          private-key: ${{ secrets.DOCS_BOT_APP_PRIVATE_KEY }}
+          owner: seqeralabs
+          repositories: docs
+
+      - name: Create Pull Request
+        uses: peter-evans/create-pull-request@v7
+        with:
+          token: ${{ steps.generate-token.outputs.token }}
+          branch-token: ${{ steps.generate-token.outputs.token }}
+          path: seqeralabs-docs
+          commit-message: "Changelog: Nextflow ${{ github.event.release.name || inputs.release_name }}"
+          title: "Changelog: Nextflow ${{ github.event.release.name || inputs.release_name }}"
+          body: |
+            This PR adds the changelog for Nextflow ${{ github.event.release.name || inputs.release_name }} to the Seqera documentation.
+
+            This is an automated PR created from the Nextflow repository.
+          branch: changelog-nextflow-${{ github.event.release.name || inputs.release_name }}
+          base: master
+          delete-branch: true

VERSION

@@ -1 +1 @@
-24.10.0
+24.11.0-edge

build.gradle

@@ -109,8 +109,8 @@ allprojects {
 
         // Documentation required libraries
         groovyDoc 'org.fusesource.jansi:jansi:2.4.0'
-        groovyDoc "org.apache.groovy:groovy-groovydoc:4.0.23"
-        groovyDoc "org.apache.groovy:groovy-ant:4.0.23"
+        groovyDoc "org.apache.groovy:groovy-groovydoc:4.0.24"
+        groovyDoc "org.apache.groovy:groovy-ant:4.0.24"
     }
 
     test {

changelog.txt

@@ -1,5 +1,61 @@
 NEXTFLOW CHANGE-LOG
 ===================
+24.11.0-edge - 3 Dec 2024
+- Add GHA to submit dependencies to dependabot (#5440) [80395a6d]
+- Add NotFoundException to retry condition for Google Batch [aa4d19cc]
+- Add Rahel Hirsch to run name generator (#5442) [ff2bc6ae]
+- Add `env()` function (#5506) [fa0e8e0f]
+- Add more scientists to run name generator (#5447) [38d9eda0]
+- Add `singularity.libraryDir` to config page (#5498) [b5e31bb0]
+- Add RepositoryProvider.revision now public property (#5500) [f0a4c526]
+- Deprecate process `shell` block (#5508) [6f527551]
+- Detecting errors in data unstaging (#5345) [3c8e602d]
+- Disable virtual threads on CI tests [ci slip] [69d07dbc]
+- Fix Fargate warning on memory check (#5475) [bdf0ad00]
+- Fix `isContainerReady` when wave is disabled (#5509) [3215afa8]
+- Fix missing wave response (#5547) [ee252173]
+- Fix overlapping conda lock file (#5540) [9248c04d]
+- Fix overlapping conda lock exception (#5489) [eaaeb3de]
+- Fix possible deadlock in dynamic `maxRetry` resolution (#5474) [25bbb621]
+- Fix Wave GCP integration test (#5490) [ad56c89b]
+- Fixing bug when execution with stub and no stub defined (#5473) [f7fd56db]
+- Fix Incorrect CPU value in Azure example (#5549) [fc5e2c2a]
+- Improve docs for using the GPU accelerator directive (#5488) [4b908524]
+- Improve groupTuple docs with scatter/gather example (#5520) [b5c63a9f]
+- Prevent NPE with null AWS Batch response [12fc1d60]
+- Target Java 17 as minimal Java version (#5045) [0140f954]
+- Update 'nexus-staging' plugin to latest version (#5462) [07934513]
+- Update gradle 'shadow' plugin version to 8.3.5 (#5463) [2a5f15f0]
+- Update install docs to reflect change from 'all' to 'dist' (#5496) [c9115659]
+- Update process snippets to comply with strict syntax (#5526) [be1694bf]
+- Update Wave dependencies [09ccd295]
+- Bump aws sdk 1.12.777 (#5458) [8bad0b4b]
+- Bump bouncycastle to jdk18on:1.78.1  (#5467) [cd8c385f]
+- Bump com.azure:azure-identity from 1.11.3 to 1.12.2 (#5449) [cb70f1df]
+- Bump commons-io:2.15.1 [767e4a0a]
+- Bump groovy 4.0.24 [dd71ad31]
+- Bump netty-common to version 4.1.115.Final [d1bbd3d0]
+- Bump nf-amazon@2.10.0 [2b653b07]
+- Bump nf-azure@1.11.0 [6af7198d]
+- Bump nf-google@1.16.0 [9494f970]
+- Bump nf-google@1.8.0 [7e2c8d82]
+- Bump protobuf-java:3.25.5 to nf-google [488b7906]
+
+24.10.2 - 27 Nov 2024
+- Prevent NPE with null AWS Batch response [3d491934]
+- Fix overlapping conda lock file (#5540) [df66deaa]
+- Fix missing wave response (#5547) [eb85cda8]
+- Bump nf-wave@1.7.4 [93d09404]
+- Bump nf-amazon@2.9.2 [469a35dd]
+
+24.10.1 - 18 Nov 2024
+- Fix overlapping file lock exception (#5489) [a2566d54]
+- Fix isContainerReady when wave is disabled (#5509) [c69e3711]
+- Bump nf-wave@1.7.3 [e7709a0f]
+- Bump nf-azure@1.10.2 [54496ac4]
+- Bump nf-amazon@2.9.1 [fa227933]
+- Bump netty-common to version 4.1.115.Final [90623c1e]
+
 24.10.0 - 27 Oct 2024
 - Add `manifest.contributors` config option (#5322) [cf0f9690]
 - Add wave mirror and scan config [92e69776]

docs/_static/dag.mmd

@@ -1,3 +1,17 @@
+%%{
+    init: {
+        'theme': 'base',
+        'themeVariables': {
+            'primaryColor': '#B6ECE2',
+            'primaryTextColor': '#160F26',
+            'primaryBorderColor': '#065647',
+            'lineColor': '#545555',
+            'clusterBkg': '#BABCBD22',
+            'clusterBorder': '#DDDEDE',
+            'fontFamily': 'arial'
+        }
+    }
+}%%
 flowchart TB
     subgraph " "
     v0["Channel.fromFilePairs"]

docs/aws.md

@@ -539,7 +539,8 @@ See the AWS documentation for details how to create the required AWS Batch queue
 and the Batch Execution Role.
 
 :::{note}
-This feature requires the use {ref}`Wave <wave-page>` container provisioning service.
+Nextflow uses [s5cmd](https://github.com/peak/s5cmd) to download the task input data and upload the task outputs.
+To enable this capability, you need to enable the Wave service in the Nextflow configuration, as shown in the above example. See {ref}`Wave <wave-page>` documentation for more details.
 :::
 
 ## Advanced configuration

docs/azure.md

@@ -155,7 +155,7 @@ Replacing `<PIPELINE NAME>` with a pipeline name e.g. `nextflow-io/rnaseq-nf` an
 
 See the [Batch documentation](https://docs.microsoft.com/en-us/azure/batch/quick-create-portal) for further details about the configuration for Azure Batch.
 
-### Pools configuration
+### Autopools
 
 When using the `autoPoolMode` option, Nextflow automatically creates a `pool` of compute nodes appropriate for your pipeline.
 
@@ -167,12 +167,12 @@ To specify multiple Azure machine families, use a comma separated list with glob
 process.machineType = "Standard_D*d_v5,Standard_E*d_v5"
 ```
 
-For example, the following process will create a pool of `Standard_E4d_v5` machines based when using `autoPoolMode`:
+For example, the following process will create a pool of `Standard_E8d_v5` machines based when using `autoPoolMode`:
 
 ```nextflow
 process EXAMPLE_PROCESS {
     machineType "Standard_E*d_v5"
-    cpus 16
+    cpus 8
     memory 8.GB
 
     script:

modules/nextflow/build.gradle

@@ -21,11 +21,11 @@ dependencies {
     api(project(':nf-commons'))
     api(project(':nf-httpfs'))
     api "org.apache.groovy:groovy:4.0.24"
-    api "org.apache.groovy:groovy-nio:4.0.23"
-    api "org.apache.groovy:groovy-xml:4.0.23"
-    api "org.apache.groovy:groovy-json:4.0.23"
-    api "org.apache.groovy:groovy-templates:4.0.23"
-    api "org.apache.groovy:groovy-yaml:4.0.23"
+    api "org.apache.groovy:groovy-nio:4.0.24"
+    api "org.apache.groovy:groovy-xml:4.0.24"
+    api "org.apache.groovy:groovy-json:4.0.24"
+    api "org.apache.groovy:groovy-templates:4.0.24"
+    api "org.apache.groovy:groovy-yaml:4.0.24"
     api "org.slf4j:jcl-over-slf4j:2.0.7"
     api "org.slf4j:jul-to-slf4j:2.0.7"
     api "org.slf4j:log4j-over-slf4j:2.0.7"
@@ -39,7 +39,7 @@ dependencies {
     api "com.beust:jcommander:1.35"
     api("com.esotericsoftware.kryo:kryo:2.24.0") { exclude group: 'com.esotericsoftware.minlog', module: 'minlog' }
     api('org.iq80.leveldb:leveldb:0.12')
-    api('org.eclipse.jgit:org.eclipse.jgit:6.10.0.202406032230-r')
+    api('org.eclipse.jgit:org.eclipse.jgit:7.1.0.202411261347-r')
     api ('javax.activation:activation:1.1.1')
     api ('javax.mail:mail:1.4.7')
     api ('org.yaml:snakeyaml:2.2')
@@ -53,7 +53,7 @@ dependencies {
     testImplementation 'org.subethamail:subethasmtp:3.1.7'
 
     // test configuration
-    testFixturesApi ("org.apache.groovy:groovy-test:4.0.23") { exclude group: 'org.apache.groovy' }
+    testFixturesApi ("org.apache.groovy:groovy-test:4.0.24") { exclude group: 'org.apache.groovy' }
     testFixturesApi ("org.objenesis:objenesis:3.4")
     testFixturesApi ("net.bytebuddy:byte-buddy:1.14.17")
     testFixturesApi ("org.spockframework:spock-core:2.3-groovy-4.0") { exclude group: 'org.apache.groovy' }

modules/nextflow/src/main/groovy/nextflow/executor/SimpleFileCopyStrategy.groovy

@@ -183,7 +183,7 @@ class SimpleFileCopyStrategy implements ScriptFileCopyStrategy {
         return """\
             IFS=\$'\\n'
             for name in \$(eval "ls -1d ${escape.join(' ')}" | sort | uniq); do
-                ${stageOutCommand('$name', targetDir, mode)} || true
+                ${stageOutCommand('$name', targetDir, mode)}
             done
             unset IFS""".stripIndent(true)
     }

modules/nextflow/src/main/groovy/nextflow/scm/AssetManager.groovy

@@ -833,7 +833,7 @@ class AssetManager {
 
     protected Map refToMap(Ref ref, Map<String,Ref> remote) {
         final entry = new HashMap(2)
-        final peel = git.getRepository().peel(ref)
+        final peel = git.getRepository().getRefDatabase().peel(ref)
         final objId = peel.getPeeledObjectId() ?: peel.getObjectId()
         // the branch or tag name
         entry.name = shortenRefName(ref.name)
@@ -867,7 +867,7 @@ class AssetManager {
         result << (name == current ? '*' : ' ')
 
         if( level ) {
-            def peel = git.getRepository().peel(ref)
+            def peel = git.getRepository().getRefDatabase().peel(ref)
             def obj = peel.getPeeledObjectId() ?: peel.getObjectId()
             result << ' '
             result << formatObjectId(obj, level == 1)