Fix plugin secrets in config

docs/developer/index.md

@@ -137,6 +137,13 @@ cd tests/checks
 ./qrun.sh
 ```
 
+To run a specific integration test:
+
+```bash
+cd tests/checks
+./qrun.sh <folder>
+```
+
 To test the documentation snippets:
 
 ```bash

docs/secrets.md

@@ -13,6 +13,8 @@ This feature allows you to decouple the use of secrets in your pipelines from th
 
 When a pipeline is launched, Nextflow injects the secrets into the run without leaking them into temporary execution files. Secrets are provided to tasks as environment variables.
 
+Secrets can be used with the local executor and the grid executors (e.g., Slurm or Grid Engine). Secrets can be used with the AWS Batch executor when launched from [Seqera Platform](https://seqera.io/blog/pipeline-secrets-secure-handling-of-sensitive-information-in-tower/).
+
 ## Command line
 
 The Nextflow {ref}`cli-secrets` sub-command can be used to manage secrets:
@@ -48,6 +50,29 @@ The above snippet accesses the secrets `MY_ACCESS_KEY` and `MY_SECRET_KEY` and a
 Secrets cannot be assigned to pipeline parameters.
 :::
 
+:::{versionadded} 25.10.0
+:::
+
+Nextflow supports the use of secrets provided by plugins (e.g., AWS secrets) in configuration. However, due to the way that plugins are loaded, there are specific considerations when using config secrets:
+
+- **Initial config load**: Nextflow first loads the configuration _without_ secrets enabled. Any reference to a secret will return the empty string `''`.
+
+- **Plugin resolution**: Plugins are resolved after the initial configuration load. This is because the configuration can specify additional plugins.
+
+- **Config reloading**: If secrets are accessed during configuration and the initial load succeeds, Nextflow will reload the configuration with secrets enabled.
+
+As a result, config secrets must be used in a way that does not cause the config resolution to fail when secrets are not present.
+
+For example:
+
+```groovy
+includeConfig secrets.MY_SECRET
+    ? "https://example.com/extra.config?secret=${secrets.MY_SECRET}"
+    : '/dev/null'
+```
+
+The above snippet includes a secured config only if the secret is present. Otherwise, it includes `/dev/null`, which is equivalent to including an empty file. The reference to `secrets.MY_SECRET` in the condition causes the config to be reloaded with secrets enabled, including secrets from plugins such as AWS secrets.
+
 (secrets-pipeline-script)=
 
 ## Pipeline script
@@ -67,10 +92,6 @@ workflow {
 The above example is only meant to demonstrate how to access a secret, not how to use it. In practice, sensitive information should not be printed to the console or output files.
 :::
 
-:::{note}
-Secrets can only be used with the local or grid executors (e.g., Slurm or Grid Engine). Secrets can be used with the AWS Batch executor when launched from [Seqera Platform](https://seqera.io/blog/pipeline-secrets-secure-handling-of-sensitive-information-in-tower/).
-:::
-
 ## Process directive
 
 Secrets can be accesses by processes using the {ref}`process-secret` directive. For example:
@@ -92,7 +113,3 @@ In the above example, the secrets `MY_ACCESS_KEY` and `MY_SECRET_KEY` are inject
 :::{warning}
 Secrets are made available as environment variables in the process script. To prevent evaluation in the Nextflow script context, escape variable names with a backslash (e.g., `\$MY_ACCESS_KEY`) as shown above.
 :::
-
-:::{note}
-Secrets can only be used with the local or grid executors (e.g., Slurm or Grid Engine). Secrets can be used with the AWS Batch executor when launched from [Seqera Platform](https://seqera.io/blog/pipeline-secrets-secure-handling-of-sensitive-information-in-tower/).
-:::

modules/nextflow/src/main/groovy/nextflow/cli/CmdRun.groovy

@@ -44,6 +44,8 @@ import nextflow.plugin.Plugins
 import nextflow.scm.AssetManager
 import nextflow.script.ScriptFile
 import nextflow.script.ScriptRunner
+import nextflow.secret.ConfigNullProvider
+import nextflow.secret.SecretsLoader
 import nextflow.util.CustomPoolFactory
 import nextflow.util.Duration
 import nextflow.util.HistoryFile
@@ -320,25 +322,39 @@ class CmdRun extends CmdBase implements HubOptions {
         checkRunName()
 
         printBanner()
-        Plugins.init()
 
-        // -- specify the arguments
+        // -- resolve main script
         final scriptFile = getScriptFile(pipeline)
 
-        // create the config object
-        final builder = new ConfigBuilder()
+        // -- load config (without secrets)
+        final secretsProvider = new ConfigNullProvider()
+        ConfigBuilder builder = new ConfigBuilder()
+            .setOptions(launcher.options)
+            .setCmdRun(this)
+            .setBaseDir(scriptFile.parent)
+            .setSecretsProvider(secretsProvider)
+        ConfigMap config = builder.build()
+
+        // -- load plugins
+        Plugins.init()
+        Plugins.load(config)
+
+        // -- load secrets provider
+        SecretsLoader.getInstance().load()
+
+        // -- reload config if secrets were used
+        if( secretsProvider.usedSecrets() ) {
+            log.debug "Config file used secrets -- reloading config with secrets provider"
+            builder = new ConfigBuilder()
                 .setOptions(launcher.options)
                 .setCmdRun(this)
                 .setBaseDir(scriptFile.parent)
-        final config = builder .build()
+            config = builder.build()
+        }
 
         // check DSL syntax in the config
         launchInfo(config, scriptFile)
 
-        // -- load plugins
-        final cfg = plugins ? [plugins: plugins.tokenize(',')] : config
-        Plugins.load(cfg)
-
         // -- validate config options
         if( NF.isSyntaxParserV2() )
             new ConfigValidator().validate(config)

modules/nextflow/src/main/groovy/nextflow/config/ConfigBuilder.groovy

@@ -33,6 +33,7 @@ import nextflow.cli.CmdRun
 import nextflow.exception.AbortOperationException
 import nextflow.exception.ConfigParseException
 import nextflow.secret.SecretsLoader
+import nextflow.secret.SecretsProvider
 import nextflow.trace.GraphObserver
 import nextflow.trace.ReportObserver
 import nextflow.trace.TimelineObserver
@@ -77,6 +78,8 @@ class ConfigBuilder {
 
     boolean showMissingVariables
 
+    SecretsProvider secretsProvider
+
     Map<ConfigObject, String> emptyVariables = new LinkedHashMap<>(10)
 
     Map<String,String> env = new HashMap<>(System.getenv())
@@ -103,6 +106,11 @@ class ConfigBuilder {
         return this
     }
 
+    ConfigBuilder setSecretsProvider(SecretsProvider value) {
+        this.secretsProvider = value
+        return this
+    }
+
     ConfigBuilder setOptions( CliOptions options ) {
         this.options = options
         return this
@@ -327,17 +335,20 @@ class ConfigBuilder {
         // this is needed to make sure to reuse the same
         // instance of the config vars across different instances of the ConfigBuilder
         // and prevent multiple parsing of the same params file (which can even be remote resource)
-        return cacheableConfigVars(baseDir)
+        final secretContext = secretsProvider
+            ? SecretsLoader.secretContext(secretsProvider)
+            : SecretsLoader.secretContext()
+        return cacheableConfigVars(baseDir, secretContext)
     }
 
     @Memoized
-    static private Map cacheableConfigVars(Path base) {
+    static private Map cacheableConfigVars(Path base, Object secretContext) {
         final binding = new HashMap(10)
         binding.put('baseDir', base)
         binding.put('projectDir', base)
         binding.put('launchDir', Paths.get('.').toRealPath())
         binding.put('outputDir', Paths.get('results').complete())
-        binding.put('secrets', SecretsLoader.secretContext())
+        binding.put('secrets', secretContext)
         return binding
     }
 
@@ -549,6 +560,9 @@ class ConfigBuilder {
         if( cmdRun.preview )
             config.preview = cmdRun.preview
 
+        if( cmdRun.plugins )
+            config.plugins = cmdRun.plugins.tokenize(',')
+
         // -- sets the working directory
         if( cmdRun.workDir )
             config.workDir = cmdRun.workDir

modules/nextflow/src/main/groovy/nextflow/secret/ConfigNullProvider.groovy

@@ -0,0 +1,42 @@
+/*
+ * Copyright 2013-2024, Seqera Labs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package nextflow.secret
+
+import groovy.transform.CompileStatic
+
+/**
+ * Specialization of the null secrets provider that is used to
+ * determine whether secrets are required in the config.
+ *
+ * @author Ben Sherman <bentshermann@gmail.com>
+ */
+@CompileStatic
+class ConfigNullProvider extends NullProvider {
+
+    private boolean accessed
+
+    @Override
+    Secret getSecret(String name) {
+        accessed = true
+        return new SecretImpl(name, '')
+    }
+
+    boolean usedSecrets() {
+        return accessed
+    }
+}

modules/nextflow/src/main/groovy/nextflow/secret/SecretsLoader.groovy

@@ -78,5 +78,9 @@ class SecretsLoader {
         final provider = isEnabled() ? getInstance().load() : new NullProvider()
         return makeSecretsContext(provider)
     }
-
+
+    static Object secretContext(SecretsProvider provider) {
+        return makeSecretsContext(provider)
+    }
+
 }

tests/checks/config-secrets.nf/.checks

@@ -0,0 +1,11 @@
+
+set -e
+
+$NXF_CMD secrets set MY_SECRET hello-world
+
+$NXF_RUN -c ../../config-secrets.config | tee stdout
+
+< .nextflow.log grep "Config file used secrets -- reloading config with secrets provider" || false
+< stdout grep "outputDir: results-hello-world" || false
+
+$NXF_CMD secrets delete MY_SECRET

tests/config-secrets.config

@@ -0,0 +1,2 @@
+
+outputDir = secrets.MY_SECRET ? "results-$secrets.MY_SECRET" : 'results'

tests/config-secrets.nf

@@ -0,0 +1,4 @@
+
+workflow {
+    println "outputDir: ${workflow.outputDir.name}"
+}