diff --git a/docs/developer/index.md b/docs/developer/index.md index a39a747e75..a27cd62258 100644 --- a/docs/developer/index.md +++ b/docs/developer/index.md @@ -137,6 +137,13 @@ cd tests/checks ./qrun.sh ``` +To run a specific integration test: + +```bash +cd tests/checks +./qrun.sh +``` + To test the documentation snippets: ```bash diff --git a/docs/secrets.md b/docs/secrets.md index 8b523a8c7d..24d4f5c32b 100644 --- a/docs/secrets.md +++ b/docs/secrets.md @@ -13,6 +13,8 @@ This feature allows you to decouple the use of secrets in your pipelines from th When a pipeline is launched, Nextflow injects the secrets into the run without leaking them into temporary execution files. Secrets are provided to tasks as environment variables. +Secrets can be used with the local executor and the grid executors (e.g., Slurm or Grid Engine). Secrets can be used with the AWS Batch executor when launched from [Seqera Platform](https://seqera.io/blog/pipeline-secrets-secure-handling-of-sensitive-information-in-tower/). + ## Command line The Nextflow {ref}`cli-secrets` sub-command can be used to manage secrets: @@ -48,6 +50,29 @@ The above snippet accesses the secrets `MY_ACCESS_KEY` and `MY_SECRET_KEY` and a Secrets cannot be assigned to pipeline parameters. ::: +:::{versionadded} 25.10.0 +::: + +Nextflow supports the use of secrets provided by plugins (e.g., AWS secrets) in configuration. However, due to the way that plugins are loaded, there are specific considerations when using config secrets: + +- **Initial config load**: Nextflow first loads the configuration _without_ secrets enabled. Any reference to a secret will return the empty string `''`. + +- **Plugin resolution**: Plugins are resolved after the initial configuration load. This is because the configuration can specify additional plugins. + +- **Config reloading**: If secrets are accessed during configuration and the initial load succeeds, Nextflow will reload the configuration with secrets enabled. + +As a result, config secrets must be used in a way that does not cause the config resolution to fail when secrets are not present. + +For example: + +```groovy +includeConfig secrets.MY_SECRET + ? "https://example.com/extra.config?secret=${secrets.MY_SECRET}" + : '/dev/null' +``` + +The above snippet includes a secured config only if the secret is present. Otherwise, it includes `/dev/null`, which is equivalent to including an empty file. The reference to `secrets.MY_SECRET` in the condition causes the config to be reloaded with secrets enabled, including secrets from plugins such as AWS secrets. + (secrets-pipeline-script)= ## Pipeline script @@ -67,10 +92,6 @@ workflow { The above example is only meant to demonstrate how to access a secret, not how to use it. In practice, sensitive information should not be printed to the console or output files. ::: -:::{note} -Secrets can only be used with the local or grid executors (e.g., Slurm or Grid Engine). Secrets can be used with the AWS Batch executor when launched from [Seqera Platform](https://seqera.io/blog/pipeline-secrets-secure-handling-of-sensitive-information-in-tower/). -::: - ## Process directive Secrets can be accesses by processes using the {ref}`process-secret` directive. For example: @@ -92,7 +113,3 @@ In the above example, the secrets `MY_ACCESS_KEY` and `MY_SECRET_KEY` are inject :::{warning} Secrets are made available as environment variables in the process script. To prevent evaluation in the Nextflow script context, escape variable names with a backslash (e.g., `\$MY_ACCESS_KEY`) as shown above. ::: - -:::{note} -Secrets can only be used with the local or grid executors (e.g., Slurm or Grid Engine). Secrets can be used with the AWS Batch executor when launched from [Seqera Platform](https://seqera.io/blog/pipeline-secrets-secure-handling-of-sensitive-information-in-tower/). -::: diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/CmdRun.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/CmdRun.groovy index d096e678e6..91d9415cb9 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/CmdRun.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/CmdRun.groovy @@ -44,6 +44,8 @@ import nextflow.plugin.Plugins import nextflow.scm.AssetManager import nextflow.script.ScriptFile import nextflow.script.ScriptRunner +import nextflow.secret.ConfigNullProvider +import nextflow.secret.SecretsLoader import nextflow.util.CustomPoolFactory import nextflow.util.Duration import nextflow.util.HistoryFile @@ -320,25 +322,39 @@ class CmdRun extends CmdBase implements HubOptions { checkRunName() printBanner() - Plugins.init() - // -- specify the arguments + // -- resolve main script final scriptFile = getScriptFile(pipeline) - // create the config object - final builder = new ConfigBuilder() + // -- load config (without secrets) + final secretsProvider = new ConfigNullProvider() + ConfigBuilder builder = new ConfigBuilder() + .setOptions(launcher.options) + .setCmdRun(this) + .setBaseDir(scriptFile.parent) + .setSecretsProvider(secretsProvider) + ConfigMap config = builder.build() + + // -- load plugins + Plugins.init() + Plugins.load(config) + + // -- load secrets provider + SecretsLoader.getInstance().load() + + // -- reload config if secrets were used + if( secretsProvider.usedSecrets() ) { + log.debug "Config file used secrets -- reloading config with secrets provider" + builder = new ConfigBuilder() .setOptions(launcher.options) .setCmdRun(this) .setBaseDir(scriptFile.parent) - final config = builder .build() + config = builder.build() + } // check DSL syntax in the config launchInfo(config, scriptFile) - // -- load plugins - final cfg = plugins ? [plugins: plugins.tokenize(',')] : config - Plugins.load(cfg) - // -- validate config options if( NF.isSyntaxParserV2() ) new ConfigValidator().validate(config) diff --git a/modules/nextflow/src/main/groovy/nextflow/config/ConfigBuilder.groovy b/modules/nextflow/src/main/groovy/nextflow/config/ConfigBuilder.groovy index 0341f73523..97c86d1c2e 100644 --- a/modules/nextflow/src/main/groovy/nextflow/config/ConfigBuilder.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/config/ConfigBuilder.groovy @@ -33,6 +33,7 @@ import nextflow.cli.CmdRun import nextflow.exception.AbortOperationException import nextflow.exception.ConfigParseException import nextflow.secret.SecretsLoader +import nextflow.secret.SecretsProvider import nextflow.trace.GraphObserver import nextflow.trace.ReportObserver import nextflow.trace.TimelineObserver @@ -77,6 +78,8 @@ class ConfigBuilder { boolean showMissingVariables + SecretsProvider secretsProvider + Map emptyVariables = new LinkedHashMap<>(10) Map env = new HashMap<>(System.getenv()) @@ -103,6 +106,11 @@ class ConfigBuilder { return this } + ConfigBuilder setSecretsProvider(SecretsProvider value) { + this.secretsProvider = value + return this + } + ConfigBuilder setOptions( CliOptions options ) { this.options = options return this @@ -327,17 +335,20 @@ class ConfigBuilder { // this is needed to make sure to reuse the same // instance of the config vars across different instances of the ConfigBuilder // and prevent multiple parsing of the same params file (which can even be remote resource) - return cacheableConfigVars(baseDir) + final secretContext = secretsProvider + ? SecretsLoader.secretContext(secretsProvider) + : SecretsLoader.secretContext() + return cacheableConfigVars(baseDir, secretContext) } @Memoized - static private Map cacheableConfigVars(Path base) { + static private Map cacheableConfigVars(Path base, Object secretContext) { final binding = new HashMap(10) binding.put('baseDir', base) binding.put('projectDir', base) binding.put('launchDir', Paths.get('.').toRealPath()) binding.put('outputDir', Paths.get('results').complete()) - binding.put('secrets', SecretsLoader.secretContext()) + binding.put('secrets', secretContext) return binding } @@ -549,6 +560,9 @@ class ConfigBuilder { if( cmdRun.preview ) config.preview = cmdRun.preview + if( cmdRun.plugins ) + config.plugins = cmdRun.plugins.tokenize(',') + // -- sets the working directory if( cmdRun.workDir ) config.workDir = cmdRun.workDir diff --git a/modules/nextflow/src/main/groovy/nextflow/secret/ConfigNullProvider.groovy b/modules/nextflow/src/main/groovy/nextflow/secret/ConfigNullProvider.groovy new file mode 100644 index 0000000000..1841e556a8 --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/secret/ConfigNullProvider.groovy @@ -0,0 +1,42 @@ +/* + * Copyright 2013-2024, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package nextflow.secret + +import groovy.transform.CompileStatic + +/** + * Specialization of the null secrets provider that is used to + * determine whether secrets are required in the config. + * + * @author Ben Sherman + */ +@CompileStatic +class ConfigNullProvider extends NullProvider { + + private boolean accessed + + @Override + Secret getSecret(String name) { + accessed = true + return new SecretImpl(name, '') + } + + boolean usedSecrets() { + return accessed + } +} diff --git a/modules/nextflow/src/main/groovy/nextflow/secret/SecretsLoader.groovy b/modules/nextflow/src/main/groovy/nextflow/secret/SecretsLoader.groovy index 2da21446b3..8f0854e1fd 100644 --- a/modules/nextflow/src/main/groovy/nextflow/secret/SecretsLoader.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/secret/SecretsLoader.groovy @@ -78,5 +78,9 @@ class SecretsLoader { final provider = isEnabled() ? getInstance().load() : new NullProvider() return makeSecretsContext(provider) } - + + static Object secretContext(SecretsProvider provider) { + return makeSecretsContext(provider) + } + } diff --git a/tests/checks/config-secrets.nf/.checks b/tests/checks/config-secrets.nf/.checks new file mode 100644 index 0000000000..78aced1791 --- /dev/null +++ b/tests/checks/config-secrets.nf/.checks @@ -0,0 +1,11 @@ + +set -e + +$NXF_CMD secrets set MY_SECRET hello-world + +$NXF_RUN -c ../../config-secrets.config | tee stdout + +< .nextflow.log grep "Config file used secrets -- reloading config with secrets provider" || false +< stdout grep "outputDir: results-hello-world" || false + +$NXF_CMD secrets delete MY_SECRET diff --git a/tests/config-secrets.config b/tests/config-secrets.config new file mode 100644 index 0000000000..eeabeef26f --- /dev/null +++ b/tests/config-secrets.config @@ -0,0 +1,2 @@ + +outputDir = secrets.MY_SECRET ? "results-$secrets.MY_SECRET" : 'results' diff --git a/tests/config-secrets.nf b/tests/config-secrets.nf new file mode 100644 index 0000000000..bb4b808145 --- /dev/null +++ b/tests/config-secrets.nf @@ -0,0 +1,4 @@ + +workflow { + println "outputDir: ${workflow.outputDir.name}" +}