Find a referenced schema next to the schema that references it

CHANGELOG.md

@@ -1,5 +1,11 @@
 # nextflow-io/nf-schema: Changelog
 
+# Version 3.1.0
+
+## Bug fixes
+
+1. Fixed an issue where a schema referenced with the `schema` keyword was only looked up relative to the project that is running. It is now looked up next to the schema that references it first, and relative to the project otherwise, so a pipeline schema keeps working when it is read from another directory (e.g. when the pipeline is included in another one).
+
 # Version 3.0.0
 
 This version contains some breaking changes to the nf-schema API. See the [migration guide](https://nextflow-io.github.io/nf-schema/3.0.0/3_0_0_migration_guide) for more information.

src/main/groovy/nextflow/validation/ValidationExtension.groovy

@@ -116,7 +116,7 @@ class ValidationExtension extends PluginExtensionPoint {
             jsonObj = input
         }
         JSONObject schemaJson = new JSONObject(schema.text)
-        ValidationResult result = validator.validate(jsonObj, schemaJson)
+        ValidationResult result = validator.validate(jsonObj, schemaJson, schema)
         List<String> errors = result.getErrors('object')
         if (exitOnError && errors != []) {
             Map<String, String> colors = getLogColors(config.monochromeLogs)

src/main/groovy/nextflow/validation/parameters/ParameterValidator.groovy

@@ -165,8 +165,9 @@ class ParameterValidator {
         Map<String,String> colors = getLogColors(config.monochromeLogs)
 
         // Validate
-        JSONObject schemaJson = new JSONObject(getBasePath(session.baseDir, schemaFilename).text)
-        ValidationResult validationResult = validator.validate(paramsJSON, schemaJson)
+        Path schemaPath = getBasePath(session.baseDir, schemaFilename)
+        JSONObject schemaJson = new JSONObject(schemaPath.text)
+        ValidationResult validationResult = validator.validate(paramsJSON, schemaJson, schemaPath)
         List<String> paramErrors = validationResult.getErrors('parameter')
         errors.addAll(paramErrors)
 

src/main/groovy/nextflow/validation/samplesheet/SamplesheetConverter.groovy

@@ -90,7 +90,7 @@ class SamplesheetConverter {
             samplesheetList = samplesheetListTemp
         }
         JSONArray samplesheet = fileToJson(samplesheetList) as JSONArray
-        ValidationResult validationResult = validator.validate(samplesheet, schemaJson)
+        ValidationResult validationResult = validator.validate(samplesheet, schemaJson, schemaFile)
         List<String> validationErrors = validationResult.getErrors('field')
         if (validationErrors) {
             /* groovylint-disable-next-line LineLength */

src/main/groovy/nextflow/validation/utils/Common.groovy

@@ -33,6 +33,18 @@ public class Common {
         return baseDir.resolve(schemaFilename.toString())
     }
 
+    // A schema referenced from another schema is looked up next to the schema that references it first,
+    // then relative to the project, so that a schema keeps working when its pipeline is run from elsewhere
+    static Path getReferencedSchemaPath(String schemaDir, Path baseDir, CharSequence schemaFilename) {
+        if (schemaDir && !Path.of(schemaFilename.toString()).absolute) {
+            Path nextToSchema = Path.of(schemaDir, schemaFilename.toString())
+            if (nextToSchema.exists()) {
+                return nextToSchema
+            }
+        }
+        return getBasePath(baseDir, schemaFilename)
+    }
+
     //
     // Function to get the value from a JSON pointer
     //

src/main/groovy/nextflow/validation/validators/JsonSchemaValidator.groovy

@@ -5,6 +5,8 @@ import static nextflow.validation.utils.Common.getValueFromJsonPointer
 
 import groovy.util.logging.Slf4j
 import groovy.transform.CompileStatic
+import java.nio.file.FileSystems
+import java.nio.file.Path
 import org.json.JSONObject
 import dev.harrel.jsonschema.ValidatorFactory
 import dev.harrel.jsonschema.Validator
@@ -28,20 +30,29 @@ import nextflow.validation.validators.evaluators.CustomEvaluatorFactory
 public class JsonSchemaValidator {
 
     final private ValidatorFactory validator
+    final private CustomEvaluatorFactory customEvaluators
     final private ValidationConfig config
 
     JsonSchemaValidator(ValidationConfig config) {
+        this.customEvaluators = new CustomEvaluatorFactory(config)
         this.validator = new ValidatorFactory()
             .withJsonNodeFactory(new OrgJsonNode.Factory())
             // .withDialect() // TODO define the dialect
             .withEvaluatorFactory(
-                EvaluatorFactory.compose(new CustomEvaluatorFactory(config), new FormatEvaluatorFactory())
+                EvaluatorFactory.compose(this.customEvaluators, new FormatEvaluatorFactory())
             )
         this.config = config
     }
 
-    ValidationResult validate(Object input, JSONObject schema) {
+    //
+    // The location of the schema is used to find the schemas it references (with the `schema` keyword)
+    // next to it
+    //
+    ValidationResult validate(Object input, JSONObject schema, Path schemaPath = null) {
         JsonNode jsonInput = new OrgJsonNode.Factory().wrap(input)
+        this.customEvaluators.schemaDir = schemaPath?.fileSystem == FileSystems.default ?
+            schemaPath.toAbsolutePath().parent?.toString() :
+            null
         return validateObject(jsonInput, input, schema)
     }
 

src/main/groovy/nextflow/validation/validators/evaluators/CustomEvaluatorFactory.groovy

@@ -22,6 +22,8 @@ class CustomEvaluatorFactory implements EvaluatorFactory {
 
     final private ValidationConfig config
     final private Path baseDir
+    // directory of the schema being validated, for the schemas it references
+    String schemaDir
 
     CustomEvaluatorFactory(ValidationConfig configInput) {
         Session session = Global.session as Session
@@ -46,7 +48,9 @@ class CustomEvaluatorFactory implements EvaluatorFactory {
         } else if (fieldName == 'exists' && schemaNode.boolean) {
             return Optional.of((Evaluator) new ExistsEvaluator(schemaNode.asBoolean()))
         } else if (fieldName == 'schema' && schemaNode.string) {
-            return Optional.of((Evaluator) new SchemaEvaluator(schemaNode.asString(), this.baseDir, this.config))
+            return Optional.of(
+                (Evaluator) new SchemaEvaluator(schemaNode.asString(), this.baseDir, this.schemaDir, this.config)
+            )
         } else if (fieldName == 'uniqueEntries' && schemaNode.array) {
             return Optional.of((Evaluator) new UniqueEntriesEvaluator(schemaNode.asArray()))
         } else if (fieldName == 'type' && (schemaNode.string || schemaNode.array) && config.lenientMode) {

src/main/groovy/nextflow/validation/validators/evaluators/SchemaEvaluator.groovy

@@ -1,6 +1,6 @@
 package nextflow.validation.validators.evaluators
 
-import static nextflow.validation.utils.Common.getBasePath
+import static nextflow.validation.utils.Common.getReferencedSchemaPath
 import static nextflow.validation.utils.FilesHelper.fileToJson
 import static nextflow.validation.utils.FilesHelper.fileToObject
 
@@ -29,10 +29,12 @@ class SchemaEvaluator implements Evaluator {
 
     private final String schema
     private final Path baseDir
+    private final String schemaDir
     private final ValidationConfig config
 
-    SchemaEvaluator(String schema, Path baseDir, ValidationConfig config) {
+    SchemaEvaluator(String schema, Path baseDir, String schemaDir, ValidationConfig config) {
         this.baseDir = baseDir
+        this.schemaDir = schemaDir
         this.schema = schema
         this.config = config
     }
@@ -64,12 +66,13 @@ class SchemaEvaluator implements Evaluator {
 
         log.debug("Started validating ${file}")
 
-        JSONObject schemaJson = new JSONObject(getBasePath(this.baseDir, this.schema).text)
+        Path schemaPath = getReferencedSchemaPath(this.schemaDir, this.baseDir, this.schema)
+        JSONObject schemaJson = new JSONObject(schemaPath.text)
         Object groovyObject = fileToObject(file, schemaJson)
         Object json = fileToJson(groovyObject)
         JsonSchemaValidator validator = new JsonSchemaValidator(config)
 
-        ValidationResult validationResult = validator.validate(json, schemaJson)
+        ValidationResult validationResult = validator.validate(json, schemaJson, schemaPath)
         List<String> validationErrors = validationResult.getErrors((json in JSONObject) ? 'parameter' : 'field')
         if (validationErrors) {
             List<String> errors = ['Validation of file failed:'] +

src/test/groovy/nextflow/validation/ValidateParametersTest.groovy

@@ -275,6 +275,118 @@ class ValidateParametersTest extends Dsl2Spec {
         !stdout
     }
 
+    void 'should find a referenced schema next to the schema that references it'() {
+        given:
+        Path dir = Files.createTempDirectory('nf-schema-referenced')
+        String parameters = new File('src/testResources/nextflow_schema_with_samplesheet.json').text
+            .replace('src/testResources/samplesheet_schema.json', 'samplesheet_schema.json')
+        new File(dir.toFile(), 'nextflow_schema.json').text = parameters
+        new File(dir.toFile(), 'samplesheet_schema.json').text = new File('src/testResources/samplesheet_schema.json').text
+        String schema = dir.resolve('nextflow_schema.json')
+        String wrongCsv = Path.of('src/testResources/wrong.csv').toAbsolutePath()
+        String script = """
+            include { validateParameters } from 'plugin/nf-schema'
+            workflow {
+                params.input = '${wrongCsv}'
+                params.outdir = 'src/testResources/testDir'
+                validateParameters(parameters_schema: '${schema}')
+            }
+        """
+
+        when:
+        Map opts = ['config': ['validation': ['monochromeLogs': true]]]
+        runScript(opts, script)
+
+        then:
+        SchemaValidationException error = thrown(SchemaValidationException)
+        error.message.readLines()[2].startsWith('* --input')
+        error.message.contains("Error for field 'strandedness' (weird)")
+
+        cleanup:
+        dir.toFile().deleteDir()
+    }
+
+    void 'should find a referenced schema relative to the project when it is not next to the schema'() {
+        given:
+        Path dir = Files.createTempDirectory('nf-schema-referenced')
+        String schema = dir.resolve('nextflow_schema.json')
+        new File(schema).text = new File('src/testResources/nextflow_schema_with_samplesheet.json').text
+        String wrongCsv = Path.of('src/testResources/wrong.csv').toAbsolutePath()
+        String script = """
+            include { validateParameters } from 'plugin/nf-schema'
+            workflow {
+                params.input = '${wrongCsv}'
+                params.outdir = 'src/testResources/testDir'
+                validateParameters(parameters_schema: '${schema}')
+            }
+        """
+
+        when:
+        Map opts = ['config': ['validation': ['monochromeLogs': true]]]
+        runScript(opts, script)
+
+        then:
+        SchemaValidationException error = thrown(SchemaValidationException)
+        error.message.contains("Error for field 'strandedness' (weird)")
+
+        cleanup:
+        dir.toFile().deleteDir()
+    }
+
+    void 'should find a schema referenced by a referenced schema next to the schema that references it'() {
+        given:
+        Path dir = Files.createTempDirectory('nf-schema-referenced')
+        String draft = 'https://json-schema.org/draft/2020-12/schema'
+        String extraCsv = dir.resolve('extra.csv')
+        String samplesheet = dir.resolve('samplesheet.csv')
+        new File(dir.toFile(), 'nextflow_schema.json').text = """{
+            "\$schema": "${draft}",
+            "type": "object",
+            "properties": { "input": { "type": "string", "format": "file-path", "schema": "samplesheet_schema.json" } }
+        }"""
+        new File(dir.toFile(), 'samplesheet_schema.json').text = """{
+            "\$schema": "${draft}",
+            "type": "array",
+            "items": {
+                "type": "object",
+                "properties": {
+                    "sample": { "type": "string" },
+                    "extra": { "type": "string", "format": "file-path", "schema": "extra_schema.json" }
+                },
+                "required": ["sample"]
+            }
+        }"""
+        new File(dir.toFile(), 'extra_schema.json').text = """{
+            "\$schema": "${draft}",
+            "type": "array",
+            "items": {
+                "type": "object",
+                "properties": { "count": { "type": "integer" } },
+                "required": ["count"]
+            }
+        }"""
+        new File(extraCsv).text = 'count\nnot_a_number\n'
+        new File(samplesheet).text = "sample,extra\ns1,${extraCsv}\n"
+        String script = """
+            include { validateParameters } from 'plugin/nf-schema'
+            workflow {
+                params.input = '${samplesheet}'
+                validateParameters(parameters_schema: '${dir.resolve('nextflow_schema.json')}')
+            }
+        """
+
+        when:
+        Map opts = ['config': ['validation': ['monochromeLogs': true]]]
+        runScript(opts, script)
+
+        then:
+        SchemaValidationException error = thrown(SchemaValidationException)
+        error.message.contains("Error for field 'count' (not_a_number)")
+
+        cleanup:
+        dir.toFile().deleteDir()
+    }
+
     void 'should validate a schema with failures - TSV'() {
         given:
         String schema = Path.of('src/testResources/nextflow_schema_with_samplesheet.json').toAbsolutePath()