diff --git a/CHANGELOG.md b/CHANGELOG.md index b0602875..b302ea55 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # nextflow-io/nf-schema: Changelog +# Version 3.1.0 + +## Bug fixes + +1. Fixed an issue where a schema referenced with the `schema` keyword was only looked up relative to the project that is running. It is now looked up next to the schema that references it first, and relative to the project otherwise, so a pipeline schema keeps working when it is read from another directory (e.g. when the pipeline is included in another one). + # Version 3.0.0 This version contains some breaking changes to the nf-schema API. See the [migration guide](https://nextflow-io.github.io/nf-schema/3.0.0/3_0_0_migration_guide) for more information. diff --git a/build.gradle b/build.gradle index 599e40a9..50ff4de8 100644 --- a/build.gradle +++ b/build.gradle @@ -11,7 +11,7 @@ dependencies { testRuntimeOnly 'org.junit.platform:junit-platform-launcher:6.1.1' } -version = '3.0.0' +version = '3.1.0' nextflowPlugin { nextflowVersion = '26.04.0' diff --git a/src/main/groovy/nextflow/validation/ValidationExtension.groovy b/src/main/groovy/nextflow/validation/ValidationExtension.groovy index a763d2b9..c9c66797 100644 --- a/src/main/groovy/nextflow/validation/ValidationExtension.groovy +++ b/src/main/groovy/nextflow/validation/ValidationExtension.groovy @@ -116,7 +116,7 @@ class ValidationExtension extends PluginExtensionPoint { jsonObj = input } JSONObject schemaJson = new JSONObject(schema.text) - ValidationResult result = validator.validate(jsonObj, schemaJson) + ValidationResult result = validator.validate(jsonObj, schemaJson, schema) List errors = result.getErrors('object') if (exitOnError && errors != []) { Map colors = getLogColors(config.monochromeLogs) diff --git a/src/main/groovy/nextflow/validation/parameters/ParameterValidator.groovy b/src/main/groovy/nextflow/validation/parameters/ParameterValidator.groovy index 34d4ee2b..431c2ae2 100644 --- a/src/main/groovy/nextflow/validation/parameters/ParameterValidator.groovy +++ b/src/main/groovy/nextflow/validation/parameters/ParameterValidator.groovy @@ -165,8 +165,9 @@ class ParameterValidator { Map colors = getLogColors(config.monochromeLogs) // Validate - JSONObject schemaJson = new JSONObject(getBasePath(session.baseDir, schemaFilename).text) - ValidationResult validationResult = validator.validate(paramsJSON, schemaJson) + Path schemaPath = getBasePath(session.baseDir, schemaFilename) + JSONObject schemaJson = new JSONObject(schemaPath.text) + ValidationResult validationResult = validator.validate(paramsJSON, schemaJson, schemaPath) List paramErrors = validationResult.getErrors('parameter') errors.addAll(paramErrors) diff --git a/src/main/groovy/nextflow/validation/samplesheet/SamplesheetConverter.groovy b/src/main/groovy/nextflow/validation/samplesheet/SamplesheetConverter.groovy index 5a25f338..207135c9 100644 --- a/src/main/groovy/nextflow/validation/samplesheet/SamplesheetConverter.groovy +++ b/src/main/groovy/nextflow/validation/samplesheet/SamplesheetConverter.groovy @@ -90,7 +90,7 @@ class SamplesheetConverter { samplesheetList = samplesheetListTemp } JSONArray samplesheet = fileToJson(samplesheetList) as JSONArray - ValidationResult validationResult = validator.validate(samplesheet, schemaJson) + ValidationResult validationResult = validator.validate(samplesheet, schemaJson, schemaFile) List validationErrors = validationResult.getErrors('field') if (validationErrors) { /* groovylint-disable-next-line LineLength */ diff --git a/src/main/groovy/nextflow/validation/utils/Common.groovy b/src/main/groovy/nextflow/validation/utils/Common.groovy index 2fa80d94..73e30538 100644 --- a/src/main/groovy/nextflow/validation/utils/Common.groovy +++ b/src/main/groovy/nextflow/validation/utils/Common.groovy @@ -33,6 +33,18 @@ public class Common { return baseDir.resolve(schemaFilename.toString()) } + // A schema referenced from another schema is looked up next to the schema that references it first, + // then relative to the project, so that a schema keeps working when its pipeline is run from elsewhere + static Path getReferencedSchemaPath(String schemaDir, Path baseDir, CharSequence schemaFilename) { + if (schemaDir && !Path.of(schemaFilename.toString()).absolute) { + Path nextToSchema = Path.of(schemaDir, schemaFilename.toString()) + if (nextToSchema.exists()) { + return nextToSchema + } + } + return getBasePath(baseDir, schemaFilename) + } + // // Function to get the value from a JSON pointer // diff --git a/src/main/groovy/nextflow/validation/validators/JsonSchemaValidator.groovy b/src/main/groovy/nextflow/validation/validators/JsonSchemaValidator.groovy index 54322f33..b56ccd22 100644 --- a/src/main/groovy/nextflow/validation/validators/JsonSchemaValidator.groovy +++ b/src/main/groovy/nextflow/validation/validators/JsonSchemaValidator.groovy @@ -5,6 +5,8 @@ import static nextflow.validation.utils.Common.getValueFromJsonPointer import groovy.util.logging.Slf4j import groovy.transform.CompileStatic +import java.nio.file.FileSystems +import java.nio.file.Path import org.json.JSONObject import dev.harrel.jsonschema.ValidatorFactory import dev.harrel.jsonschema.Validator @@ -28,20 +30,29 @@ import nextflow.validation.validators.evaluators.CustomEvaluatorFactory public class JsonSchemaValidator { final private ValidatorFactory validator + final private CustomEvaluatorFactory customEvaluators final private ValidationConfig config JsonSchemaValidator(ValidationConfig config) { + this.customEvaluators = new CustomEvaluatorFactory(config) this.validator = new ValidatorFactory() .withJsonNodeFactory(new OrgJsonNode.Factory()) // .withDialect() // TODO define the dialect .withEvaluatorFactory( - EvaluatorFactory.compose(new CustomEvaluatorFactory(config), new FormatEvaluatorFactory()) + EvaluatorFactory.compose(this.customEvaluators, new FormatEvaluatorFactory()) ) this.config = config } - ValidationResult validate(Object input, JSONObject schema) { + // + // The location of the schema is used to find the schemas it references (with the `schema` keyword) + // next to it + // + ValidationResult validate(Object input, JSONObject schema, Path schemaPath = null) { JsonNode jsonInput = new OrgJsonNode.Factory().wrap(input) + this.customEvaluators.schemaDir = schemaPath?.fileSystem == FileSystems.default ? + schemaPath.toAbsolutePath().parent?.toString() : + null return validateObject(jsonInput, input, schema) } diff --git a/src/main/groovy/nextflow/validation/validators/evaluators/CustomEvaluatorFactory.groovy b/src/main/groovy/nextflow/validation/validators/evaluators/CustomEvaluatorFactory.groovy index a1477848..1702043d 100644 --- a/src/main/groovy/nextflow/validation/validators/evaluators/CustomEvaluatorFactory.groovy +++ b/src/main/groovy/nextflow/validation/validators/evaluators/CustomEvaluatorFactory.groovy @@ -22,6 +22,8 @@ class CustomEvaluatorFactory implements EvaluatorFactory { final private ValidationConfig config final private Path baseDir + // directory of the schema being validated, for the schemas it references + String schemaDir CustomEvaluatorFactory(ValidationConfig configInput) { Session session = Global.session as Session @@ -46,7 +48,9 @@ class CustomEvaluatorFactory implements EvaluatorFactory { } else if (fieldName == 'exists' && schemaNode.boolean) { return Optional.of((Evaluator) new ExistsEvaluator(schemaNode.asBoolean())) } else if (fieldName == 'schema' && schemaNode.string) { - return Optional.of((Evaluator) new SchemaEvaluator(schemaNode.asString(), this.baseDir, this.config)) + return Optional.of( + (Evaluator) new SchemaEvaluator(schemaNode.asString(), this.baseDir, this.schemaDir, this.config) + ) } else if (fieldName == 'uniqueEntries' && schemaNode.array) { return Optional.of((Evaluator) new UniqueEntriesEvaluator(schemaNode.asArray())) } else if (fieldName == 'type' && (schemaNode.string || schemaNode.array) && config.lenientMode) { diff --git a/src/main/groovy/nextflow/validation/validators/evaluators/SchemaEvaluator.groovy b/src/main/groovy/nextflow/validation/validators/evaluators/SchemaEvaluator.groovy index 510c4bb1..194325ee 100644 --- a/src/main/groovy/nextflow/validation/validators/evaluators/SchemaEvaluator.groovy +++ b/src/main/groovy/nextflow/validation/validators/evaluators/SchemaEvaluator.groovy @@ -1,6 +1,6 @@ package nextflow.validation.validators.evaluators -import static nextflow.validation.utils.Common.getBasePath +import static nextflow.validation.utils.Common.getReferencedSchemaPath import static nextflow.validation.utils.FilesHelper.fileToJson import static nextflow.validation.utils.FilesHelper.fileToObject @@ -29,10 +29,12 @@ class SchemaEvaluator implements Evaluator { private final String schema private final Path baseDir + private final String schemaDir private final ValidationConfig config - SchemaEvaluator(String schema, Path baseDir, ValidationConfig config) { + SchemaEvaluator(String schema, Path baseDir, String schemaDir, ValidationConfig config) { this.baseDir = baseDir + this.schemaDir = schemaDir this.schema = schema this.config = config } @@ -64,12 +66,13 @@ class SchemaEvaluator implements Evaluator { log.debug("Started validating ${file}") - JSONObject schemaJson = new JSONObject(getBasePath(this.baseDir, this.schema).text) + Path schemaPath = getReferencedSchemaPath(this.schemaDir, this.baseDir, this.schema) + JSONObject schemaJson = new JSONObject(schemaPath.text) Object groovyObject = fileToObject(file, schemaJson) Object json = fileToJson(groovyObject) JsonSchemaValidator validator = new JsonSchemaValidator(config) - ValidationResult validationResult = validator.validate(json, schemaJson) + ValidationResult validationResult = validator.validate(json, schemaJson, schemaPath) List validationErrors = validationResult.getErrors((json in JSONObject) ? 'parameter' : 'field') if (validationErrors) { List errors = ['Validation of file failed:'] + diff --git a/src/test/groovy/nextflow/validation/ValidateParametersTest.groovy b/src/test/groovy/nextflow/validation/ValidateParametersTest.groovy index 365da4df..e3d27d48 100644 --- a/src/test/groovy/nextflow/validation/ValidateParametersTest.groovy +++ b/src/test/groovy/nextflow/validation/ValidateParametersTest.groovy @@ -275,6 +275,118 @@ class ValidateParametersTest extends Dsl2Spec { !stdout } + void 'should find a referenced schema next to the schema that references it'() { + given: + Path dir = Files.createTempDirectory('nf-schema-referenced') + String parameters = new File('src/testResources/nextflow_schema_with_samplesheet.json').text + .replace('src/testResources/samplesheet_schema.json', 'samplesheet_schema.json') + new File(dir.toFile(), 'nextflow_schema.json').text = parameters + new File(dir.toFile(), 'samplesheet_schema.json').text = new File('src/testResources/samplesheet_schema.json').text + String schema = dir.resolve('nextflow_schema.json') + String wrongCsv = Path.of('src/testResources/wrong.csv').toAbsolutePath() + String script = """ + include { validateParameters } from 'plugin/nf-schema' + workflow { + params.input = '${wrongCsv}' + params.outdir = 'src/testResources/testDir' + validateParameters(parameters_schema: '${schema}') + } + """ + + when: + Map opts = ['config': ['validation': ['monochromeLogs': true]]] + runScript(opts, script) + + then: + SchemaValidationException error = thrown(SchemaValidationException) + error.message.readLines()[2].startsWith('* --input') + error.message.contains("Error for field 'strandedness' (weird)") + + cleanup: + dir.toFile().deleteDir() + } + + void 'should find a referenced schema relative to the project when it is not next to the schema'() { + given: + Path dir = Files.createTempDirectory('nf-schema-referenced') + String schema = dir.resolve('nextflow_schema.json') + new File(schema).text = new File('src/testResources/nextflow_schema_with_samplesheet.json').text + String wrongCsv = Path.of('src/testResources/wrong.csv').toAbsolutePath() + String script = """ + include { validateParameters } from 'plugin/nf-schema' + workflow { + params.input = '${wrongCsv}' + params.outdir = 'src/testResources/testDir' + validateParameters(parameters_schema: '${schema}') + } + """ + + when: + Map opts = ['config': ['validation': ['monochromeLogs': true]]] + runScript(opts, script) + + then: + SchemaValidationException error = thrown(SchemaValidationException) + error.message.contains("Error for field 'strandedness' (weird)") + + cleanup: + dir.toFile().deleteDir() + } + + void 'should find a schema referenced by a referenced schema next to the schema that references it'() { + given: + Path dir = Files.createTempDirectory('nf-schema-referenced') + String draft = 'https://json-schema.org/draft/2020-12/schema' + String extraCsv = dir.resolve('extra.csv') + String samplesheet = dir.resolve('samplesheet.csv') + new File(dir.toFile(), 'nextflow_schema.json').text = """{ + "\$schema": "${draft}", + "type": "object", + "properties": { "input": { "type": "string", "format": "file-path", "schema": "samplesheet_schema.json" } } + }""" + new File(dir.toFile(), 'samplesheet_schema.json').text = """{ + "\$schema": "${draft}", + "type": "array", + "items": { + "type": "object", + "properties": { + "sample": { "type": "string" }, + "extra": { "type": "string", "format": "file-path", "schema": "extra_schema.json" } + }, + "required": ["sample"] + } + }""" + new File(dir.toFile(), 'extra_schema.json').text = """{ + "\$schema": "${draft}", + "type": "array", + "items": { + "type": "object", + "properties": { "count": { "type": "integer" } }, + "required": ["count"] + } + }""" + new File(extraCsv).text = 'count\nnot_a_number\n' + new File(samplesheet).text = "sample,extra\ns1,${extraCsv}\n" + String script = """ + include { validateParameters } from 'plugin/nf-schema' + workflow { + params.input = '${samplesheet}' + validateParameters(parameters_schema: '${dir.resolve('nextflow_schema.json')}') + } + """ + + when: + Map opts = ['config': ['validation': ['monochromeLogs': true]]] + runScript(opts, script) + + then: + SchemaValidationException error = thrown(SchemaValidationException) + error.message.contains("Error for field 'count' (not_a_number)") + + cleanup: + dir.toFile().deleteDir() + } + void 'should validate a schema with failures - TSV'() { given: String schema = Path.of('src/testResources/nextflow_schema_with_samplesheet.json').toAbsolutePath()