Enable multiple revisions for project repos (first attempt 2024)

docs/cli.md

@@ -366,7 +366,7 @@ The `clone` command downloads a pipeline from a Git-hosting platform into the *c
 : Service hub where the project is hosted. Options: `gitlab` or `bitbucket`.
 
 `-r` (`master`)
-: Revision to clone - It can be a git branch, tag, or revision number.
+: Revision of the project to clone (either a git branch, tag or commit SHA number).
 
 `-user`
 : Private repository user name.
@@ -415,6 +415,9 @@ The `config` command is used for printing the project's configuration i.e. the `
 `-properties`
 : Print config using Java properties notation.
 
+`-r, -revision`
+: Revision of the project (either a git branch, tag or commit SHA number).
+
 `-a, -show-profiles`
 : Show all configuration profiles.
 
@@ -535,12 +538,18 @@ The `drop` command is used to remove the projects which have been downloaded int
 
 **Options**
 
+`-a, -all-revisions`
+: For specified project, drop all revisions.
+
 `-f`
 : Delete the repository without taking care of local changes.
 
 `-h, -help`
 : Print the command usage.
 
+`-r, -revision`
+: Revision of the project to drop (either a git branch, tag or commit SHA number).
+
 **Examples**
 
 Drop the `nextflow-io/hello` project.
@@ -664,7 +673,7 @@ $ nextflow info [options] [project]
 
 **Description**
 
-The `info` command prints out the nextflow runtime information about the hardware as well as the software versions of the Nextflow version and build, operating system, and Groovy and Java runtime. It can also be used to display information about a specific project.
+The `info` command prints out the nextflow runtime information about the hardware as well as the software versions of the Nextflow version and build, operating system, and Groovy and Java runtime. It can also be used to display information about a specific project; in this case, note how revisions marked as `P` are pulled locally.
 
 If no run name or session id is provided, it will clean the latest run.
 
@@ -682,6 +691,9 @@ If no run name or session id is provided, it will clean the latest run.
 `-o` (`text`)
 : Output format, either `text`, `json` or `yaml`.
 
+`-r, -revision`
+: Revision of the project (either a git branch, tag or commit SHA number).
+
 **Examples**
 
 Display Nextflow runtime and system info:
@@ -878,6 +890,9 @@ The `list` commands prints a list of the projects which are already downloaded i
 
 **Options**
 
+`-a, -all-revisions`
+: For each project, also list revisions.
+
 `-h, -help`
 : Print the command usage.
 
@@ -1072,7 +1087,7 @@ The `pull` command downloads a pipeline from a Git-hosting platform into the glo
 : Service hub where the project is hosted. Options: `gitlab` or `bitbucket`
 
 `-r, -revision`
-: Revision of the project to run (either a git branch, tag or commit hash).
+: Revision of the project to pull (either a git branch, tag or commit SHA number).
 : When passing a git tag or branch, the `workflow.revision` and `workflow.commitId` fields are populated. When passing only the commit hash, `workflow.revision` is not defined.
 
 `-user`
@@ -1215,7 +1230,7 @@ The `run` command is used to execute a local pipeline script or remote pipeline
 : Execute the script using the cached results, useful to continue executions that was stopped by an error.
 
 `-r, -revision`
-: Revision of the project to run (either a git branch, tag or commit hash).
+: Revision of the project to run (either a git branch, tag or commit SHA number).
 : When passing a git tag or branch, the `workflow.revision` and `workflow.commitId` fields are populated. When passing only the commit hash, `workflow.revision` is not defined.
 
 `-stub-run, -stub`
@@ -1424,6 +1439,9 @@ The `view` command is used to inspect the pipelines that are already stored in t
 `-q`
 : Hide header line.
 
+`-r, -revision`
+: Revision of the project (either a git branch, tag or commit SHA number).
+
 **Examples**
 
 Viewing the contents of a downloaded pipeline.

docs/developer/diagrams/nextflow.scm.mmd

@@ -8,6 +8,7 @@ classDiagram
 
     class AssetManager {
         project : String
+        revision : String
         localPath : File
         mainScript : String
         repositoryProvider : RepositoryProvider

modules/nextflow/src/main/groovy/nextflow/cli/CmdClone.groovy

@@ -15,6 +15,9 @@
  */
 
 package nextflow.cli
+
+import static nextflow.scm.AssetManager.revisionDelim
+
 import com.beust.jcommander.Parameter
 import com.beust.jcommander.Parameters
 import groovy.transform.CompileStatic
@@ -37,7 +40,7 @@ class CmdClone extends CmdBase implements HubOptions {
     @Parameter(required=true, description = 'name of the project to clone')
     List<String> args
 
-    @Parameter(names='-r', description = 'Revision to clone - It can be a git branch, tag or revision number')
+    @Parameter(names='-r', description = 'Revision of the project to clone (either a git branch, tag or commit SHA number)')
     String revision
 
     @Parameter(names=['-d','-deep'], description = 'Create a shallow clone of the specified depth')
@@ -52,7 +55,8 @@ class CmdClone extends CmdBase implements HubOptions {
         Plugins.init()
         // the pipeline name
         String pipeline = args[0]
-        final manager = new AssetManager(pipeline, this)
+        String revisionSuffix = revision ? revisionDelim + revision : ''
+        final manager = new AssetManager(pipeline, revision, this)
 
         // the target directory is the second parameter
         // otherwise default the current pipeline name
@@ -68,9 +72,9 @@ class CmdClone extends CmdBase implements HubOptions {
         }
 
         manager.checkValidRemoteRepo()
-        print "Cloning ${manager.project}${revision ? ':'+revision:''} ..."
-        manager.clone(target, revision, deep)
+        print "Cloning ${manager.project}${revisionSuffix} ..."
+        manager.clone(target, deep)
         print "\r"
-        println "${manager.project} cloned to: $target"
+        println "${manager.project}${revisionSuffix} cloned to: $target"
     }
 }

modules/nextflow/src/main/groovy/nextflow/cli/CmdConfig.groovy

@@ -44,6 +44,9 @@ class CmdConfig extends CmdBase {
     @Parameter(description = 'project name')
     List<String> args = []
 
+    @Parameter(names=['-r','-revision'], description = 'Revision of the project (either a git branch, tag or commit SHA number)')
+    String revision
+
     @Parameter(names=['-a','-show-profiles'], description = 'Show all configuration profiles')
     boolean showAllProfiles
 
@@ -182,7 +185,7 @@ class CmdConfig extends CmdBase {
             return file.parent ?: Paths.get('/')
         }
 
-        final manager = new AssetManager(path)
+        final manager = new AssetManager(path, revision)
         manager.isLocal() ? manager.localPath.toPath() : manager.configFile?.parent
 
     }

modules/nextflow/src/main/groovy/nextflow/cli/CmdDrop.groovy

@@ -16,6 +16,8 @@
 
 package nextflow.cli
 
+import static nextflow.scm.AssetManager.revisionDelim
+
 import com.beust.jcommander.Parameter
 import com.beust.jcommander.Parameters
 import groovy.transform.CompileStatic
@@ -39,6 +41,12 @@ class CmdDrop extends CmdBase {
     @Parameter(required=true, description = 'name of the project to drop')
     List<String> args
 
+    @Parameter(names=['-r','-revision'], description = 'Revision of the project to drop (either a git branch, tag or commit SHA number)')
+    String revision
+
+    @Parameter(names=['-a','-all-revisions'], description = 'For specified project, drop all revisions')
+    Boolean allrevisions
+
     @Parameter(names='-f', description = 'Delete the repository without taking care of local changes')
     boolean force
 
@@ -48,18 +56,34 @@ class CmdDrop extends CmdBase {
     @Override
     void run() {
         Plugins.init()
-        def manager = new AssetManager(args[0])
-        if( !manager.localPath.exists() ) {
-            throw new AbortOperationException("No match found for: ${args[0]}")
+
+        List<AssetManager> dropList = []
+        if ( allrevisions ) {
+            def referenceManager = new AssetManager(args[0])
+            referenceManager.listRevisions().each {
+                dropList << new AssetManager(it.tokenize(revisionDelim)[0], it.tokenize(revisionDelim)[1])
+            }
+        } else {
+            dropList << new AssetManager(args[0], revision)
         }
 
-        if( this.force || manager.isClean() ) {
-            manager.close()
-            if( !manager.localPath.deleteDir() )
-                throw new AbortOperationException("Unable to delete project `${manager.project}` -- Check access permissions for path: ${manager.localPath}")
-            return
+        if ( !dropList ) {
+            throw new AbortOperationException("No revisions found for specified project: ${args[0]}")
         }
 
-        throw new AbortOperationException("Local project repository contains uncommitted changes -- won't drop it")
+        dropList.each { manager ->
+            if( !manager.localPath.exists() ) {
+                throw new AbortOperationException("No match found for: ${manager.project}${manager.revision ? revisionDelim + manager.revision : ''}")
+            }
+
+            if( this.force || manager.isClean() ) {
+                manager.close()
+                if( !manager.localPath.deleteDir() )
+                    throw new AbortOperationException("Unable to delete project `${manager.project}${manager.revision ? revisionDelim + manager.revision : ''}` -- Check access permissions for path: ${manager.localPath}")
+                return
+            }
+
+            throw new AbortOperationException("Local project repository contains uncommitted changes -- won't drop it")
+        }
     }
 }

modules/nextflow/src/main/groovy/nextflow/cli/CmdInfo.groovy

@@ -16,6 +16,8 @@
 
 package nextflow.cli
 
+import static nextflow.scm.AssetManager.revisionDelim
+
 import java.lang.management.ManagementFactory
 import java.nio.file.spi.FileSystemProvider
 
@@ -50,6 +52,9 @@ class CmdInfo extends CmdBase {
     @Parameter(description = 'project name')
     List<String> args
 
+    @Parameter(names=['-r','-revision'], description = 'Revision of the project (either a git branch, tag or commit SHA number)')
+    String revision
+
     @Parameter(names='-d',description = 'Show detailed information', arity = 0)
     boolean detailed
 
@@ -75,9 +80,16 @@ class CmdInfo extends CmdBase {
         }
 
         Plugins.init()
-        final manager = new AssetManager(args[0])
-        if( !manager.isLocal() )
-            throw new AbortOperationException("Unknown project `${args[0]}`")
+        def manager = new AssetManager(args[0], revision)
+        if( !manager.isLocal() ) {
+            // if no revision specified and default branch not found locally, use first one from list of local pulls
+            if ( manager.listRevisions() && !revision ) {
+                manager = new AssetManager(args[0], manager.getPulledRevisions()[0])
+            }
+            else {
+                throw new AbortOperationException("Unknown project `${args[0]}${revision ? revisionDelim + revision : ''}`")
+            }
+        }
 
         if( !format || format == 'text' ) {
             printText(manager,level)
@@ -99,7 +111,9 @@ class CmdInfo extends CmdBase {
     protected printText(AssetManager manager, int level) {
         final manifest = manager.getManifest()
 
+        def printedRevision = manager.revision ?: manager.getDefaultBranch()
         out.println " project name: ${manager.project}"
+        out.println " revision    : ${printedRevision}"
         out.println " repository  : ${manager.repositoryUrl}"
         out.println " local path  : ${manager.localPath}"
         out.println " main script : ${manager.mainScriptName}"
@@ -137,6 +151,7 @@ class CmdInfo extends CmdBase {
     protected Map createMap(AssetManager manager) {
         def result = [:]
         result.projectName = manager.project
+        result.revision = manager.revision ?: manager.getDefaultBranch()
         result.repository = manager.repositoryUrl
         result.localPath = manager.localPath?.toString()
         result.manifest = manager.manifest.toMap()

modules/nextflow/src/main/groovy/nextflow/cli/CmdList.groovy

@@ -16,6 +16,9 @@
 
 package nextflow.cli
 
+import static nextflow.scm.AssetManager.revisionDelim
+
+import com.beust.jcommander.Parameter
 import com.beust.jcommander.Parameters
 import groovy.transform.CompileStatic
 import groovy.util.logging.Slf4j
@@ -33,6 +36,9 @@ class CmdList extends CmdBase {
 
     static final public NAME = 'list'
 
+    @Parameter(names=['-a','-all-revisions'], description = 'For each project, also list revisions')
+    Boolean revisions
+
     @Override
     final String getName() { NAME }
 
@@ -45,7 +51,15 @@ class CmdList extends CmdBase {
             return
         }
 
-        all.each { println it }
+    if (revisions) {
+        all.collect{ it.tokenize(revisionDelim) }
+           .groupBy{ it[0] }
+           .each{ println ' ' + it.value[0][0] ; it.value.each{ y -> println ( y.size()==1 ? '   (default)' : '   ' + y[1] ) } }
+    } else {
+        all.collect{ it.replaceAll( /$revisionDelim.*/, '' ) }
+           .unique()
+           .each{ println ' ' + it }
+    }
     }
 
 }

modules/nextflow/src/main/groovy/nextflow/cli/CmdPull.groovy

@@ -15,6 +15,9 @@
  */
 
 package nextflow.cli
+
+import static nextflow.scm.AssetManager.revisionDelim
+
 import com.beust.jcommander.Parameter
 import com.beust.jcommander.Parameters
 import groovy.transform.CompileStatic
@@ -40,7 +43,7 @@ class CmdPull extends CmdBase implements HubOptions {
     @Parameter(names='-all', description = 'Update all downloaded projects', arity = 0)
     boolean all
 
-    @Parameter(names=['-r','-revision'], description = 'Revision of the project to run (either a git branch, tag or commit SHA number)')
+    @Parameter(names=['-r','-revision'], description = 'Revision of the project to pull (either a git branch, tag or commit SHA number)')
     String revision
 
     @Parameter(names=['-d','-deep'], description = 'Create a shallow clone of the specified depth')
@@ -73,10 +76,10 @@ class CmdPull extends CmdBase implements HubOptions {
         Plugins.init()
 
         list.each {
-            log.info "Checking $it ..."
-            def manager = new AssetManager(it, this)
+            log.info "Checking $it${revision ? revisionDelim + revision : ''} ..."
+            def manager = new AssetManager(it, revision, this)
 
-            def result = manager.download(revision,deep)
+            def result = manager.download(deep)
             manager.updateModules()
 
             def scriptFile = manager.getScriptFile()