Unify nf-lang config scopes with runtime classes

Signed-off-by: Ben Sherman <bentshermann@gmail.com>

Author: bentsherman

docs/google.md

@@ -2,6 +2,8 @@
 
 # Google Cloud
 
+(google-credentials)=
+
 ## Credentials
 
 Credentials for submitting requests to the Google Cloud Batch API are picked up from your environment using [Application Default Credentials](https://github.com/googleapis/google-auth-library-java#google-auth-library-oauth2-http). Application Default Credentials are designed to use the credentials most natural to the environment in which a tool runs.

docs/notifications.md

@@ -189,17 +189,26 @@ mail {
 }
 ```
 
+:::{note}
+Some versions of Java (e.g. Java 11 Corretto) do not default to TLS v1.2, and as a result may have issues with 3rd party integrations that enforce TLS v1.2 (e.g. Azure Active Directory OIDC). This problem can be addressed by setting the following config option:
+
+```groovy
+mail {
+    smtp.ssl.protocols = 'TLSv1.2'
+}
+```
+:::
+
 See the {ref}`mail scope <config-mail>` section to learn more the mail server configuration options.
 
 ### AWS SES configuration
 
 :::{versionadded} 23.06.0-edge
 :::
 
-Nextflow supports [AWS SES](https://aws.amazon.com/ses/) native API as an alternative
-provider to send emails in place of SMTP server.
+Nextflow supports the [AWS Simple Email Service](https://aws.amazon.com/ses/) API as an alternative provider to send emails in place of an SMTP server.
 
-To enable this feature add the following environment variable in the launching environment:
+To enable this feature, set the following environment variable in the launch environment:
 
 ```bash
 export NXF_ENABLE_AWS_SES=true
@@ -211,6 +220,20 @@ Make also sure to add the following AWS IAM permission to the AWS user (or role)
 ses:SendRawEmail
 ```
 
+The following snippet shows how to configure Nextflow to send emails through SES:
+
+```groovy
+mail {
+    smtp.host = 'email-smtp.us-east-1.amazonaws.com'
+    smtp.port = 587
+    smtp.user = '<Your AWS SES access key>'
+    smtp.password = '<Your AWS SES secret key>'
+    smtp.auth = true
+    smtp.starttls.enable = true
+    smtp.starttls.required = true
+}
+```
+
 ## Mail notification
 
 You can use the `sendMail` function with a {ref}`workflow completion handler <metadata-completion-handler>` to notify the completion of a workflow completion. For example:

docs/reference/config.md

@@ -164,8 +164,11 @@ profiles {
 }
 ```
 
-The above configuration snippet allows the execution either with Spack or Docker by specifying `-profile spack` or
-`-profile docker` when running the pipeline script.
+The above configuration snippet allows the execution either with Spack or Docker by specifying `-profile spack` or `-profile docker` when running the pipeline script.
+
+:::{note}
+Nextflow does not allow for fine-grained configuration of the Spack package manager. Instead, this has to be performed directly on the host Spack installation. For more information see the [Spack documentation](https://spack.readthedocs.io).
+:::
 
 ## Advanced settings
 

docs/spack.md

@@ -673,7 +673,7 @@ class Channel  {
         final linExt = Plugins.getExtension(LinExtension)
         if( !linExt )
             throw new IllegalStateException("Unable to load lineage extensions.")
-        final future = CompletableFuture.runAsync(() -> linExt.fromLineage(session, channel, params))
+        final future = CompletableFuture.runAsync(() -> linExt.fromLineage(session.config, channel, params))
         future.exceptionally(this.&handlerException)
     }
 }

modules/nextflow/src/main/groovy/nextflow/Channel.groovy

@@ -348,11 +348,8 @@ class Nextflow {
      *      - attach: One or more list attachment
      */
     static void sendMail( Map params ) {
-
-        new Mailer()
-            .setConfig(Global.session.config.mail as Map)
-            .send(params)
-
+        final opts = Global.session.config.mail as Map ?: Collections.emptyMap()
+        new Mailer(opts).send(params)
     }
 
     /**
@@ -374,9 +371,8 @@ class Nextflow {
      *    <code>
      */
     static void sendMail( Closure params ) {
-        new Mailer()
-                .setConfig(Global.session.config.mail as Map)
-                .send(params)
+        final opts = Global.session.config.mail as Map ?: Collections.emptyMap()
+        new Mailer(opts).send(params)
     }
 
     /**

modules/nextflow/src/main/groovy/nextflow/Nextflow.groovy

@@ -37,7 +37,14 @@ import nextflow.cache.CacheDB
 import nextflow.cache.CacheFactory
 import nextflow.conda.CondaConfig
 import nextflow.config.Manifest
+import nextflow.container.ApptainerConfig
+import nextflow.container.CharliecloudConfig
 import nextflow.container.ContainerConfig
+import nextflow.container.DockerConfig
+import nextflow.container.PodmanConfig
+import nextflow.container.SarusConfig
+import nextflow.container.ShifterConfig
+import nextflow.container.SingularityConfig
 import nextflow.dag.DAG
 import nextflow.exception.AbortOperationException
 import nextflow.exception.AbortSignalException
@@ -436,7 +443,7 @@ class Session implements ISession {
         }
 
         // set the byte-code target directory
-        this.disableRemoteBinDir = getExecConfigProp(null, 'disableRemoteBinDir', false)
+        this.disableRemoteBinDir = (config.executor as Map)?.disableRemoteBinDir as boolean
         this.classesDir = FileHelper.createLocalDir()
         this.executorFactory = new ExecutorFactory(Plugins.manager)
         this.observersV2 = createObserversV2()
@@ -1176,14 +1183,14 @@ class Session implements ISession {
 
     @Memoized
     CondaConfig getCondaConfig() {
-        final cfg = config.conda as Map ?: Collections.emptyMap()
-        return new CondaConfig(cfg, getSystemEnv())
+        final opts = config.conda as Map ?: Collections.emptyMap()
+        return new CondaConfig(opts, getSystemEnv())
     }
 
     @Memoized
     SpackConfig getSpackConfig() {
-        final cfg = config.spack as Map ?: Collections.emptyMap()
-        return new SpackConfig(cfg, getSystemEnv())
+        final opts = config.spack as Map ?: Collections.emptyMap()
+        return new SpackConfig(opts, getSystemEnv())
     }
 
     /**
@@ -1199,91 +1206,38 @@ class Session implements ISession {
     @Memoized
     ContainerConfig getContainerConfig(String engine) {
 
-        final allEngines = new LinkedList<Map>()
-        getContainerConfig0('docker', allEngines)
-        getContainerConfig0('podman', allEngines)
-        getContainerConfig0('sarus', allEngines)
-        getContainerConfig0('shifter', allEngines)
-        getContainerConfig0('udocker', allEngines)
-        getContainerConfig0('singularity', allEngines)
-        getContainerConfig0('apptainer', allEngines)
-        getContainerConfig0('charliecloud', allEngines)
+        final allConfigs = [
+            new DockerConfig(config.docker as Map ?: Collections.emptyMap()),
+            new PodmanConfig(config.podman as Map ?: Collections.emptyMap()),
+            new SarusConfig(config.sarus as Map ?: Collections.emptyMap()),
+            new ShifterConfig(config.shifter as Map ?: Collections.emptyMap()),
+            new SingularityConfig(config.singularity as Map ?: Collections.emptyMap()),
+            new ApptainerConfig(config.apptainer as Map ?: Collections.emptyMap()),
+            new CharliecloudConfig(config.charliecloud as Map ?: Collections.emptyMap()),
+        ] as List<ContainerConfig>
 
         if( engine ) {
-            final result = allEngines.find(it -> it.engine==engine) ?: [engine: engine]
-            return new ContainerConfig(result)
+            return allConfigs.find { it -> it.engine == engine }
         }
 
-        final enabled = allEngines.findAll(it -> it.enabled?.toString() == 'true')
-        if( enabled.size() > 1 ) {
-            final names = enabled.collect(it -> it.engine)
+        final allEnabled = allConfigs.findAll { it -> it.enabled }
+        if( allEnabled.size() > 1 ) {
+            final names = allEnabled.collect { it -> it.engine }
             throw new IllegalConfigException("Cannot enable more than one container engine -- Choose either one of: ${names.join(', ')}")
         }
-        if( enabled ) {
-            return new ContainerConfig(enabled.get(0))
+        if( allEnabled ) {
+            return allEnabled.first()
         }
-        if( allEngines ) {
-            return new ContainerConfig(allEngines.get(0))
+        if( allConfigs ) {
+            return allConfigs.first()
         }
-        return new ContainerConfig(engine:'docker')
+        return new DockerConfig([:])
     }
 
     ContainerConfig getContainerConfig() {
         return getContainerConfig(null)
     }
 
-    private void getContainerConfig0(String engine, List<Map> drivers) {
-        assert engine
-        final entry = this.config?.get(engine)
-        if( entry instanceof Map ) {
-            final config0 = new LinkedHashMap()
-            config0.putAll((Map)entry)
-            config0.put('engine', engine)
-            drivers.add(config0)
-        }
-        else if( entry!=null ) {
-            log.warn "Malformed configuration for container engine '$engine' -- One or more attributes should be provided"
-        }
-    }
-
-    @Memoized
-    def getExecConfigProp( String execName, String name, Object defValue, Map env = null  ) {
-        def result = ConfigHelper.getConfigProperty(config.executor, execName, name )
-        if( result != null )
-            return result
-
-        // -- try to fallback sys env
-        def key = "NXF_EXECUTOR_${name.toUpperCase().replaceAll(/\./,'_')}".toString()
-        if( env == null ) env = System.getenv()
-        return env.containsKey(key) ? env.get(key) : defValue
-    }
-
-    @Memoized
-    def getConfigAttribute(String name, defValue )  {
-        def result = getMap0(getConfig(),name,name)
-        if( result != null )
-            return result
-
-        def key = "NXF_${name.toUpperCase().replaceAll(/\./,'_')}".toString()
-        def env = getSystemEnv()
-        return (env.containsKey(key) ? env.get(key) : defValue)
-    }
-
-    private getMap0(Map map, String name, String fqn) {
-        def p=name.indexOf('.')
-        if( p == -1 )
-            return map.get(name)
-        else {
-            def k=name.substring(0,p)
-            def v=map.get(k)
-            if( v == null )
-                return null
-            if( v instanceof Map )
-                return getMap0(v,name.substring(p+1),fqn)
-            throw new IllegalArgumentException("Not a valid config attribute: $fqn -- Missing element: $k")
-        }
-    }
-
     @Memoized
     protected Map<String,String> getSystemEnv() {
         new HashMap<String, String>(System.getenv())
@@ -1350,68 +1304,6 @@ class Session implements ISession {
         return String.valueOf(val)
     }
 
-
-    /**
-     * Defines the number of tasks the executor will handle in a parallel manner
-     *
-     * @param execName The executor name
-     * @param defValue The default value if setting is not defined in the configuration file
-     * @return The value of tasks to handle in parallel
-     */
-    @Memoized
-    int getQueueSize( String execName, int defValue ) {
-        getExecConfigProp(execName, 'queueSize', defValue) as int
-    }
-
-    /**
-     * Determines how often a poll occurs to check for a process termination
-     *
-     * @param execName The executor name
-     * @param defValue The default value if setting is not defined in the configuration file
-     * @return A {@code Duration} object. Default '1 second'
-     */
-    @Memoized
-    Duration getPollInterval( String execName, Duration defValue = Duration.of('1sec') ) {
-        getExecConfigProp( execName, 'pollInterval', defValue ) as Duration
-    }
-
-    /**
-     *  Determines how long the executors waits before return an error status when a process is
-     *  terminated but the exit file does not exist or it is empty. This setting is used only by grid executors
-     *
-     * @param execName The executor name
-     * @param defValue The default value if setting is not defined in the configuration file
-     * @return A {@code Duration} object. Default '90 second'
-     */
-    @Memoized
-    Duration getExitReadTimeout( String execName, Duration defValue = Duration.of('90sec') ) {
-        getExecConfigProp( execName, 'exitReadTimeout', defValue ) as Duration
-    }
-
-    /**
-     * Determines how often the executor status is written in the application log file
-     *
-     * @param execName The executor name
-     * @param defValue The default value if setting is not defined in the configuration file
-     * @return A {@code Duration} object. Default '5 minutes'
-     */
-    @Memoized
-    Duration getMonitorDumpInterval( String execName, Duration defValue = Duration.of('5min')) {
-        getExecConfigProp(execName, 'dumpInterval', defValue) as Duration
-    }
-
-    /**
-     * Determines how often the queue status is fetched from the cluster system. This setting is used only by grid executors
-     *
-     * @param execName The executor name
-     * @param defValue  The default value if setting is not defined in the configuration file
-     * @return A {@code Duration} object. Default '1 minute'
-     */
-    @Memoized
-    Duration getQueueStatInterval( String execName, Duration defValue = Duration.of('1min') ) {
-        getExecConfigProp(execName, 'queueStatInterval', defValue) as Duration
-    }
-
     void printConsole(String str, boolean newLine=false) {
         if( ansiLogObserver )
             ansiLogObserver.appendInfo(str)

modules/nextflow/src/main/groovy/nextflow/Session.groovy

@@ -23,7 +23,6 @@ import com.beust.jcommander.Parameter
 import com.beust.jcommander.Parameters
 import groovy.transform.CompileStatic
 import nextflow.config.ConfigBuilder
-import nextflow.config.ConfigMap
 import nextflow.exception.AbortOperationException
 import nextflow.plugin.Plugins
 import org.pf4j.ExtensionPoint
@@ -40,12 +39,12 @@ class CmdLineage extends CmdBase implements UsageAware {
     private static final String NAME = 'lineage'
 
     interface LinCommand extends ExtensionPoint {
-        void list(ConfigMap config)
-        void view(ConfigMap config, List<String> args)
-        void render(ConfigMap config, List<String> args)
-        void diff(ConfigMap config, List<String> args)
-        void find(ConfigMap config, List<String> args)
-        void check(ConfigMap config, List<String> args)
+        void list(Map config)
+        void view(Map config, List<String> args)
+        void render(Map config, List<String> args)
+        void diff(Map config, List<String> args)
+        void find(Map config, List<String> args)
+        void check(Map config, List<String> args)
     }
 
     interface SubCmd {
@@ -59,7 +58,7 @@ class CmdLineage extends CmdBase implements UsageAware {
 
     private LinCommand operation
 
-    private ConfigMap config
+    private Map config
 
     CmdLineage() {
         commands << new CmdList()

modules/nextflow/src/main/groovy/nextflow/cli/CmdLineage.groovy

@@ -28,6 +28,7 @@ import groovy.util.logging.Slf4j
 import groovyx.gpars.dataflow.DataflowVariable
 import groovyx.gpars.dataflow.LazyDataflowVariable
 import nextflow.Global
+import nextflow.SysEnv
 import nextflow.file.FileMutex
 import nextflow.util.CacheHelper
 import nextflow.util.Duration
@@ -56,7 +57,7 @@ class CondaCache {
     /**
      * Timeout after which the environment creation is aborted
      */
-    private Duration createTimeout = Duration.of('20min')
+    private Duration createTimeout
 
     private String createOptions
 
@@ -72,7 +73,7 @@ class CondaCache {
 
     @PackageScope Duration getCreateTimeout() { createTimeout }
 
-    @PackageScope Map<String,String> getEnv() { System.getenv() }
+    @PackageScope Map<String,String> getEnv() { SysEnv.get() }
 
     @PackageScope Path getConfigCacheDir0() { configCacheDir0 }