Add support for official Nextflow parameters meta schema

CHANGELOG.md

@@ -4,8 +4,11 @@
 
 ## New features
 
-1. Added a new configuration option: `validation.maxErrValSize` which sets the maximum length that a value in an error message can be. The default is set to 150 characters.
+1. Added 2 new configuration options:
+   - `validation.maxErrValSize` which sets the maximum length that a value in an error message can be. The default is set to 150 characters.
+   - `validation.mode` which sets the validation mode. Possible options are `limited` and `unlimited`. In `limited` mode the JSON schemas for parameters will be limited to the official Nextflow specs to ensure compatibility with nf-core tooling, Seqera platform and other tools. These restrictions are not set when using `unlimited` mode. The default is set to `unlimited` to retain backwards compatibility.
 2. Added a new function: `validate()` that can be used to validate any data structure using a JSON schema.
+3. Add support for the official Nextflow JSON schema specification for parameters located in https://github.com/nextflow-io/schema-spec
 
 ## Bug fixes
 

docs/configuration/configuration.md

@@ -30,6 +30,17 @@ validation.parametersSchema = "path/to/schema.json" // default "nextflow_schema.
 
 This option can either be a path relative to the root of the pipeline directory or a full path to the JSON schema (Be wary to not use hardcoded local paths to ensure your pipeline will keep working on other systems)
 
+## mode
+
+This option can be used to set the validation mode for parameters. There are two valid options:
+
+- `unlimited`: No restrictions apply on the validation of parameters. All keywords and schema structures available in [draft 2020-12](https://json-schema.org/draft/2020-12) can be used freely.
+- `limited`: Limits the usage of keywords and schema structures for parameter validation to a subset of [draft 2020-12](https://json-schema.org/draft/2020-12). Enabling this mode will ensure compatibility with nf-core tooling, Seqera platform and others. The specified `$schema` inside of the parameters JSON schema needs to be set to the [official Nextflow meta schema](https://github.com/nextflow-io/schema-spec) for parameters.
+
+```groovy
+validation.mode = "limited|unlimited" // default: "limited"
+```
+
 ## monochromeLogs
 
 This option can be used to turn of the colored logs from nf-validation. This can be useful if you run a Nextflow pipeline in an environment that doesn't support colored logging.

docs/nextflow_schema/nextflow_schema_specification.md

@@ -82,7 +82,7 @@ However, they will be displayed as ungrouped in tools working off the schema.
 
 ## Nested parameters
 
-!!! example "New feature in v2.1.0"
+!!! example "Not supported in [limited](../configuration/configuration.md#mode) mode"
 
 Nextflow config allows parameters to be nested as objects, for example:
 
@@ -397,6 +397,8 @@ Example usage is as follows:
 
 ### `mimetype`
 
+!!! example "Not supported in [limited](../configuration/configuration.md#mode) mode"
+
 MIME type for a file path. Setting this value informs downstream tools about what _kind_ of file is expected.
 
 Should only be set when `format` is `file-path`.
@@ -465,6 +467,8 @@ Specify a minimum / maximum value for an integer or float number length with `mi
 
 ## Array-specific keys
 
+!!! example "Not supported in [limited](../configuration/configuration.md#mode) mode"
+
 ### `uniqueItems`
 
 All items in the array should be unique.

plugins/nf-schema/src/main/nextflow/validation/config/ValidationConfig.groovy

@@ -21,6 +21,7 @@ class ValidationConfig {
     final public String  parametersSchema
     final public Boolean showHiddenParams
     final public Integer maxErrValSize = 150
+    final public String  mode = "unlimited"
     final public HelpConfig help
     final public SummaryConfig summary
 
@@ -47,6 +48,16 @@ class ValidationConfig {
         help                    = new HelpConfig(config.help as Map ?: [:], params, monochromeLogs, showHiddenParams)
         summary                 = new SummaryConfig(config.summary as Map ?: [:], monochromeLogs)
 
+        if(config.containsKey("mode")) {
+            def List<String> allowedModes = ["limited", "unlimited"]
+            if(allowedModes.contains(config.mode)) {
+                mode = config.mode
+            } else {
+                log.warn("Detected an unsupported mode in `validation.mode`, supported options are: ${allowedModes}, defaulting to `${mode}`")
+            }
+            log.debug("Set nf-schema validation mode to `${mode}`")
+        }
+
         if(config.ignoreParams && !(config.ignoreParams instanceof List<String>)) {
             throw new SchemaValidationException("Config value 'validation.ignoreParams' should be a list of String values")
         }

plugins/nf-schema/src/main/nextflow/validation/validators/JsonSchemaValidator.groovy

@@ -9,7 +9,11 @@ import dev.harrel.jsonschema.EvaluatorFactory
 import dev.harrel.jsonschema.FormatEvaluatorFactory
 import dev.harrel.jsonschema.JsonNode
 import dev.harrel.jsonschema.providers.OrgJsonNode
+import dev.harrel.jsonschema.SchemaResolver
+import nextflow.Nextflow
 
+import java.nio.file.Path
+import java.nio.file.Paths
 import java.util.regex.Pattern
 import java.util.regex.Matcher
 
@@ -28,24 +32,41 @@ import nextflow.validation.validators.evaluators.CustomEvaluatorFactory
 @Slf4j
 public class JsonSchemaValidator {
 
-    private ValidatorFactory validator
-    private Pattern uriPattern = Pattern.compile('^#/(\\d*)?/?(.*)$')
+    // TODO Change the base URL once the meta schema has been officially released
+    final private Map<String,String> supportedCustomParameterDrafts = [
+        "https://github.com/nextflow-io/schema-spec/raw/refs/heads/main/parameters_meta_schema.json": "/schemas/parameters/1.0/parameters_meta_schema.json"
+    ]
+
     private ValidationConfig config
+    private ValidatorFactory validator
 
     JsonSchemaValidator(ValidationConfig config) {
+        def SchemaResolver resolver = (String uri) -> {
+            if(supportedCustomParameterDrafts.containsKey(uri)) {
+                return SchemaResolver.Result.fromString(Nextflow.file(getClass().getResource(supportedCustomParameterDrafts[uri]).getFile()).text)
+            }
+            return SchemaResolver.Result.empty()
+        }
+
         this.validator = new ValidatorFactory()
             .withJsonNodeFactory(new OrgJsonNode.Factory())
-            // .withDialect() // TODO define the dialect
+            .withSchemaResolver(resolver)
             .withEvaluatorFactory(EvaluatorFactory.compose(new CustomEvaluatorFactory(config), new FormatEvaluatorFactory()))
         this.config = config
     }
-
     private Tuple2<List<String>,List<String>> validateObject(JsonNode input, String validationType, Object rawJson, String schemaString) {
         def JSONObject schema = new JSONObject(schemaString)
         def String draft = getValueFromJsonPointer("#/\$schema", schema)
-        if(draft != "https://json-schema.org/draft/2020-12/schema") {
+        def String draft2020_12 = "https://json-schema.org/draft/2020-12/schema"
+        if(config.mode == "limited" && validationType == "parameter" && !supportedCustomParameterDrafts.containsKey(draft)) {
+            log.error("""Failed to load meta schema:
+    Using '${draft}' for parameter JSON schemas is not allowed in limited mode. Please use one the following meta schemas instead:
+${supportedCustomParameterDrafts.collect{ url, cachedSchema -> "    - ${url}"}.join("\n") }
+            """)
+            throw new SchemaValidationException("", [])
+        } else if(draft != draft2020_12 && !supportedCustomParameterDrafts.containsKey(draft)) {
             log.error("""Failed to load the meta schema:
-    The used schema draft (${draft}) is not correct, please use \"https://json-schema.org/draft/2020-12/schema\" instead.
+    The used schema draft (${draft}) is not correct, please use \"${draft2020_12}\" instead.
         - If you are a pipeline developer, check our migration guide for more information: https://nextflow-io.github.io/nf-schema/latest/migration_guide/
         - If you are a pipeline user, revert back to nf-validation to avoid this error: https://www.nextflow.io/docs/latest/plugins.html#using-plugins, i.e. set `plugins {
     id 'nf-validation@1.1.3'

plugins/nf-schema/src/resources/schemas/parameters/1.0/parameters_meta_schema.json

@@ -0,0 +1 @@
+{"$schema": "https://json-schema.org/draft/2020-12/schema","$id": "https://nextflow.io","title": "Nextflow Schema Meta-schema","description": "Meta-schema to validate Nextflow parameter schema files","type": "object","properties": {"$schema": {"title": "schema","type": "string","minLength": 1},"$id": {"title": "ID URI","type": "string","minLength": 1},"title": {"title": "Title","type": "string","minLength": 1},"description": {"title": "Description","type": "string","minLength": 1},"type": {"title": "Top level type","type": "string","const": "object"},"$defs": {"title": "Parameter groups","type": "object","patternProperties": {"^.*$": {"type": "object","required": ["title", "type", "properties"],"properties": {"title": {"type": "string","minLength": 1},"type": {"const": "object"},"fa_icon": {"type": "string","pattern": "^fa"},"description": {"type": "string"},"required": {"type": "array"},"properties": {"$ref": "#/$defs/parameterOptions"},"dependentRequired": {"type": "object","additionalProperties": {"type": "array","items": {"type": "string"},"uniqueItems": true,"default": []}}}}}},"properties": {"$ref": "#/$defs/parameterOptions"},"dependentRequired": {"type": "object","additionalProperties": {"type": "array","items": {"type": "string"},"uniqueItems": true,"default": []}},"allOf": {"title": "Combine definition groups","type": "array","items": {"type": "object","required": ["$ref"],"properties": {"$ref": {"type": "string","pattern": "^#/\\$defs/[^/]+$"}}}}},"required": ["$schema", "$id", "title", "description", "type"],"$defs": {"typeAnnotation": {"type": "string","enum": ["string", "boolean", "integer", "number", "null"]},"allTypes": {"type": ["integer", "boolean", "string", "number", "null"]},"nonNegativeInteger": {"type": "integer","minimum": 0},"parameterOptions": {"type": "object","patternProperties": {"^.*$": {"type": "object","allOf": [{ "$ref": "#/$defs/standardKeywords" },{ "$ref": "#/$defs/customKeywords" }],"required": ["type", "description"],"unevaluatedProperties": false}}},"standardKeywords": {"type": "object","description": "Allowed standard JSON Schema properties.","properties": {"type": {"description": "The type of the parameter value. Can be one or more of these: `['integer', 'boolean', 'number', 'string', 'null']`","anyOf": [{ "$ref": "#/$defs/typeAnnotation" },{"type": "array","items": { "$ref": "#/$defs/typeAnnotation" }}]},"format": {"type": "string","description": "The format of a parameter value with the 'string' type. This is used for additional validation on the structure of the value.","enum": ["file-path","directory-path","path","file-path-pattern","date-time","date","time","email","uri","regex"]},"pattern": {"type": "string","format": "regex","minLength": 1,"description": "Check a parameter value of 'string' type against a regex pattern"},"description": {"type": "string","description": "The description of the current parameter"},"default": {"$ref": "#/$defs/allTypes","description": "Specifies a default value to use for this parameter"},"examples": {"type": "array","items": {"$ref": "#/$defs/allTypes"},"description": "A list of examples for the current parameter"},"deprecated": {"type": "boolean","description": "States that the parameter is deprecated. Please provide a nice deprecation message using 'errorMessage'"},"minLength": {"$ref": "#/$defs/nonNegativeInteger","description": "The minimum length a 'string' parameter value should be"},"maxLength": {"$ref": "#/$defs/nonNegativeInteger","description": "The maximum length a 'string' parameter value should be"},"minimum": {"type": "number","description": "The mimimum value an 'integer' or 'number' parameter value should be"},"exclusiveMinimum": {"type": "number","description": "The exclusive mimimum value an 'integer' or 'number' parameter value should be"},"maximum": {"type": "number","description": "The maximum value an 'integer' or 'number' parameter value should be"},"exclusiveMaximum": {"type": "number","description": "The exclusive maximum value an 'integer' or 'number' parameter value should be"},"multipleOf": {"type": "number","description": "The 'integer' or 'number' parameter value should be a multiple of this value"},"enum": {"type": "array","uniqueItems": true,"items": {"$ref": "#/$defs/allTypes"},"description": "The parameter value should be one of the values specified in this enum array"},"const": {"$ref": "#/$defs/allTypes","description": "The parameter value should be equal to this value"}}},"customKeywords": {"type": "object","description": "Additional custom JSON Schema properties.","properties": {"errorMessage": {"type": "string","minLength": 1,"description": "NON STANDARD OPTION: The custom error message to display in case validation against this parameter fails. Can also be used as a deprecation message if 'deprecated' is true"},"exists": {"type": "boolean","description": "NON STANDARD OPTION: Check if a file exists. This parameter value needs to be a `string` type with one of these formats: `['path', 'file-path', 'directory-path', 'file-path-pattern']`"},"schema": {"type": "string","minLength": 1,"pattern": "\\.json$","description": "NON STANDARD OPTION: Check the given file against a schema passed to this keyword. Will only work when type is `string` and format is `path` or `file-path`"},"help_text": {"type": "string","description": "NON STANDARD OPTION: A more detailed help text"},"fa_icon": {"type": "string","pattern": "^fa","description": "NON STANDARD OPTION: A font awesome icon to use in the nf-core parameter documentation"},"hidden": {"type": "boolean","description": "NON STANDARD OPTION: Hide this parameter from the help message and documentation"},"mimetype": {"type": "string","description": "NON STANDARD OPTION: The MIME type of the parameter value","deprecated": true,"errorMessage": "The 'mimetype' keyword is deprecated. Use 'pattern' or 'format' instead."}},"dependentSchemas": {"exists": {"$comment": "This checks if the type is a 'string' and the format corresponds to a file or directory when 'exists' is used.","properties": {"type": { "const": "string" },"format": {"enum": ["path","file-path","file-path-pattern","directory-path"]}}},"schema": {"$comment": "This checks if the type is a 'string' and the format is 'path' or 'file-path'","properties": {"type": { "const": "string" },"format": {"enum": ["path", "file-path"]}}}}}}}