diff --git a/.gitignore b/.gitignore index b80da080..7e31b39c 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,7 @@ __pycache__/ # protocol buffers generated headers include/osmformat.pb.h include/vector_tile.pb.h +src/config_schema.h # downloaded data coastline diff --git a/CMakeLists.txt b/CMakeLists.txt index c67b73ad..0143a2d4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -92,8 +92,15 @@ else() set(THREAD_LIB pthread) endif() +file(READ "${CMAKE_CURRENT_SOURCE_DIR}/resources/config-schema.json" TILEMAKER_CONFIG_SCHEMA_JSON) +configure_file( + cmake/config_schema.h.in + "${CMAKE_BINARY_DIR}/config_schema.h" + @ONLY) + file(GLOB tilemaker_src_files src/attribute_store.cpp + src/config_validator.cpp src/coordinates.cpp src/coordinates_geom.cpp src/external/streamvbyte_decode.c diff --git a/Dockerfile b/Dockerfile index b8043eca..ec6fb477 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,6 +24,7 @@ COPY cmake ./cmake COPY src ./src COPY include ./include COPY server ./server +COPY resources/config-schema.json ./resources/config-schema.json RUN mkdir build && \ cd build && \ diff --git a/Makefile b/Makefile index b08088f2..17f35b7f 100644 --- a/Makefile +++ b/Makefile @@ -87,6 +87,7 @@ all: tilemaker server tilemaker: \ src/attribute_store.o \ + src/config_validator.o \ src/coordinates_geom.o \ src/coordinates.o \ src/external/streamvbyte_decode.o \ @@ -140,6 +141,7 @@ tilemaker: \ test: \ test_append_vector \ test_attribute_store \ + test_config_validator \ test_deque_map \ test_helpers \ test_options_parser \ @@ -164,6 +166,19 @@ test_attribute_store: \ test/attribute_store.test.o $(CXX) $(CXXFLAGS) -o test.attribute_store $^ $(INC) $(LIB) $(LDFLAGS) && ./test.attribute_store +test_config_validator: \ + src/config_validator.o \ + test/config_validator.test.o + $(CXX) $(CXXFLAGS) -o test.config_validator $^ $(INC) $(LIB) $(LDFLAGS) && ./test.config_validator + +src/config_schema.h: resources/config-schema.json + printf '#ifndef _CONFIG_SCHEMA_H\n#define _CONFIG_SCHEMA_H\n\nstatic const char* CONFIG_SCHEMA = R"TMCONFIGSCHEMA(\n' > $@ + cat $< >> $@ + printf '\n)TMCONFIGSCHEMA";\n\n#endif //_CONFIG_SCHEMA_H\n' >> $@ + +src/config_validator.o: src/config_validator.cpp include/config_validator.h src/config_schema.h + $(CXX) $(CXXFLAGS) -o $@ -c $< $(INC) + test_deque_map: \ test/deque_map.test.o $(CXX) $(CXXFLAGS) -o test.deque_map $^ $(INC) $(LIB) $(LDFLAGS) && ./test.deque_map @@ -268,6 +283,6 @@ install: @install docs/man/tilemaker.1 ${DESTDIR}${MANPREFIX}/man1/ || true clean: - rm -f tilemaker tilemaker-server src/*.o src/external/*.o src/external/libdeflate/lib/*.o src/external/libdeflate/lib/*/*.o include/*.o include/*.pb.h server/*.o test/*.o + rm -f tilemaker tilemaker-server src/*.o src/external/*.o src/external/libdeflate/lib/*.o src/external/libdeflate/lib/*/*.o include/*.o include/*.pb.h server/*.o test/*.o src/config_schema.h .PHONY: install diff --git a/cmake/config_schema.h.in b/cmake/config_schema.h.in new file mode 100644 index 00000000..5baa046c --- /dev/null +++ b/cmake/config_schema.h.in @@ -0,0 +1,8 @@ +#ifndef _CONFIG_SCHEMA_H +#define _CONFIG_SCHEMA_H + +static const char* CONFIG_SCHEMA = R"TMCONFIGSCHEMA( +@TILEMAKER_CONFIG_SCHEMA_JSON@ +)TMCONFIGSCHEMA"; + +#endif //_CONFIG_SCHEMA_H diff --git a/include/config_validator.h b/include/config_validator.h new file mode 100644 index 00000000..7cdda2a2 --- /dev/null +++ b/include/config_validator.h @@ -0,0 +1,10 @@ +#ifndef _CONFIG_VALIDATOR_H +#define _CONFIG_VALIDATOR_H + +#include + +#include "rapidjson/document.h" + +bool validateConfigJson(const rapidjson::Document &jsonConfig, std::string &error); + +#endif //_CONFIG_VALIDATOR_H diff --git a/resources/config-schema.json b/resources/config-schema.json new file mode 100644 index 00000000..34a9d1c4 --- /dev/null +++ b/resources/config-schema.json @@ -0,0 +1,90 @@ +{ + "type": "object", + "required": ["settings", "layers"], + "properties": { + "settings": { + "type": "object", + "required": ["basezoom", "minzoom", "maxzoom", "include_ids", "compress", "name", "version", "description"], + "properties": { + "basezoom": { "type": "integer", "minimum": 0 }, + "minzoom": { "type": "integer", "minimum": 0 }, + "maxzoom": { "type": "integer", "minimum": 0 }, + "include_ids": { "type": "boolean" }, + "compress": { "type": "string", "enum": ["gzip", "deflate", "none"] }, + "name": { "type": "string" }, + "version": { "type": "string" }, + "description": { "type": "string" }, + "high_resolution": { "type": "boolean" }, + "combine_below": { "type": "integer", "minimum": 0 }, + "mvt_version": { "type": "integer", "minimum": 1 }, + "bounding_box": { + "type": "array", + "minItems": 4, + "maxItems": 4, + "items": { "type": "number" } + }, + "default_view": { + "type": "array", + "minItems": 3, + "maxItems": 3, + "items": [ + { "type": "number" }, + { "type": "number" }, + { "type": "integer" } + ] + }, + "metadata": { + "type": "object", + "properties": { + "attribution": { "type": "string" } + }, + "additionalProperties": true + }, + "filemetadata": { "type": "object" } + }, + "additionalProperties": true + }, + "layers": { + "type": "object", + "additionalProperties": { + "type": "object", + "required": ["minzoom", "maxzoom"], + "properties": { + "minzoom": { "type": "integer", "minimum": 0 }, + "maxzoom": { "type": "integer", "minimum": 0 }, + "write_to": { "type": "string" }, + "simplify_below": { "type": "integer", "minimum": 0 }, + "simplify_level": { "type": "number" }, + "simplify_length": { "type": "number" }, + "simplify_ratio": { "type": "number" }, + "filter_below": { "type": "integer", "minimum": 0 }, + "filter_area": { "type": "number" }, + "feature_limit": { "type": "integer", "minimum": 0 }, + "feature_limit_below": { "type": "integer", "minimum": 0 }, + "combine_points": { "type": "boolean" }, + "combine_lines_below": { "type": "integer", "minimum": 0 }, + "combine_polygons_below": { "type": "integer", "minimum": 0 }, + "z_order_ascending": { "type": "boolean" }, + "simplify_algorithm": { "type": "string" }, + "source": { "type": "string" }, + "source_columns": { + "oneOf": [ + { + "type": "boolean", + "enum": [true] + }, + { + "type": "array", + "items": { "type": "string" } + } + ] + }, + "index": { "type": "boolean" }, + "index_column": { "type": "string" } + }, + "additionalProperties": true + } + } + }, + "additionalProperties": true +} diff --git a/src/config_validator.cpp b/src/config_validator.cpp new file mode 100644 index 00000000..25b20bd2 --- /dev/null +++ b/src/config_validator.cpp @@ -0,0 +1,145 @@ +#include "config_validator.h" + +#include + +#include +#include + +#include "rapidjson/pointer.h" +#include "rapidjson/schema.h" +#include "rapidjson/stringbuffer.h" +#include "rapidjson/writer.h" + +namespace { +std::string pointerString(const rapidjson::Pointer &pointer) { + rapidjson::StringBuffer buffer; + pointer.StringifyUriFragment(buffer); + return buffer.GetString(); +} + +std::string valueToString(const rapidjson::Value &value) { + if (value.IsString()) return std::string("\"") + value.GetString() + "\""; + + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + value.Accept(writer); + return buffer.GetString(); +} + +std::string joinValues(const std::vector &values) { + std::string joined; + for (std::size_t i = 0; i < values.size(); i++) { + if (i > 0) joined += ", "; + joined += values[i]; + } + return joined; +} + +std::string valueTypeName(const rapidjson::Value &value) { + if (value.IsNull()) return "null"; + if (value.IsBool()) return "boolean"; + if (value.IsObject()) return "object"; + if (value.IsArray()) return "array"; + if (value.IsString()) return "string"; + if (value.IsNumber()) return "number"; + return "unknown"; +} + +std::string requiredError(const rapidjson::Document &schemaJson, + const rapidjson::Document &jsonConfig, + const rapidjson::Pointer &schemaPointer, + const rapidjson::Pointer &documentPointer, + const std::string &documentPointerString) { + const rapidjson::Value* schemaNode = schemaPointer.Get(schemaJson); + const rapidjson::Value* documentNode = documentPointer.Get(jsonConfig); + if (!schemaNode || !schemaNode->IsObject() || !schemaNode->HasMember("required") || + !(*schemaNode)["required"].IsArray() || !documentNode || !documentNode->IsObject()) { + return ""; + } + + std::vector missing; + for (rapidjson::Value::ConstValueIterator it = (*schemaNode)["required"].Begin(); it != (*schemaNode)["required"].End(); ++it) { + if (it->IsString() && !documentNode->HasMember(it->GetString())) { + missing.push_back(std::string("\"") + it->GetString() + "\""); + } + } + if (missing.empty()) return ""; + + return "missing required " + std::string(missing.size() == 1 ? "field " : "fields ") + + joinValues(missing) + " at " + documentPointerString; +} + +std::string typeError(const rapidjson::Document &schemaJson, + const rapidjson::Document &jsonConfig, + const rapidjson::Pointer &schemaPointer, + const rapidjson::Pointer &documentPointer, + const std::string &documentPointerString) { + const rapidjson::Value* schemaNode = schemaPointer.Get(schemaJson); + const rapidjson::Value* documentNode = documentPointer.Get(jsonConfig); + if (!schemaNode || !schemaNode->IsObject() || !schemaNode->HasMember("type") || !documentNode) { + return ""; + } + + return "invalid type at " + documentPointerString + ": expected " + + valueToString((*schemaNode)["type"]) + ", got " + valueTypeName(*documentNode); +} + +std::string enumError(const rapidjson::Document &schemaJson, + const rapidjson::Document &jsonConfig, + const rapidjson::Pointer &schemaPointer, + const rapidjson::Pointer &documentPointer, + const std::string &documentPointerString) { + const rapidjson::Value* schemaNode = schemaPointer.Get(schemaJson); + const rapidjson::Value* documentNode = documentPointer.Get(jsonConfig); + if (!schemaNode || !schemaNode->IsObject() || !schemaNode->HasMember("enum") || + !(*schemaNode)["enum"].IsArray() || !documentNode) { + return ""; + } + + std::vector allowed; + for (rapidjson::Value::ConstValueIterator it = (*schemaNode)["enum"].Begin(); it != (*schemaNode)["enum"].End(); ++it) { + allowed.push_back(valueToString(*it)); + } + + return "invalid value at " + documentPointerString + ": expected one of " + + joinValues(allowed) + ", got " + valueToString(*documentNode); +} +} // namespace + +bool validateConfigJson(const rapidjson::Document &jsonConfig, std::string &error) { + rapidjson::Document schemaJson; + schemaJson.Parse(CONFIG_SCHEMA); + if (schemaJson.HasParseError()) { + error = "Internal config schema is invalid."; + return false; + } + + rapidjson::SchemaDocument schema(schemaJson); + rapidjson::SchemaValidator validator(schema); + if (jsonConfig.Accept(validator)) { + return true; + } + + std::string documentPointer = pointerString(validator.GetInvalidDocumentPointer()); + if (documentPointer.empty()) documentPointer = "#"; + std::string schemaPointer = pointerString(validator.GetInvalidSchemaPointer()); + if (schemaPointer.empty()) schemaPointer = "#"; + + const char* keyword = validator.GetInvalidSchemaKeyword(); + if (std::strcmp(keyword, "required") == 0) { + error = requiredError(schemaJson, jsonConfig, validator.GetInvalidSchemaPointer(), + validator.GetInvalidDocumentPointer(), documentPointer); + } else if (std::strcmp(keyword, "type") == 0) { + error = typeError(schemaJson, jsonConfig, validator.GetInvalidSchemaPointer(), + validator.GetInvalidDocumentPointer(), documentPointer); + } else if (std::strcmp(keyword, "enum") == 0) { + error = enumError(schemaJson, jsonConfig, validator.GetInvalidSchemaPointer(), + validator.GetInvalidDocumentPointer(), documentPointer); + } + + if (error.empty()) { + error = "schema validation failed at " + documentPointer + ": " + + keyword + " (" + schemaPointer + ")"; + } + return false; +} diff --git a/src/tilemaker.cpp b/src/tilemaker.cpp index 6cbd920a..ccf3bf91 100644 --- a/src/tilemaker.cpp +++ b/src/tilemaker.cpp @@ -28,6 +28,7 @@ #include "rapidjson/stringbuffer.h" #include "rapidjson/filereadstream.h" #include "rapidjson/filewritestream.h" +#include "rapidjson/error/en.h" #ifndef _MSC_VER #include @@ -38,6 +39,7 @@ #include "way_stores.h" // Tilemaker code +#include "config_validator.h" #include "helpers.h" #include "coordinates.h" #include "coordinates_geom.h" @@ -173,8 +175,18 @@ int main(const int argc, const char* argv[]) { char readBuffer[65536]; rapidjson::FileReadStream is(fp, readBuffer, sizeof(readBuffer)); jsonConfig.ParseStream(is); - if (jsonConfig.HasParseError()) { cerr << "Invalid JSON file." << endl; return -1; } fclose(fp); + if (jsonConfig.HasParseError()) { + cerr << "Invalid JSON file: " << rapidjson::GetParseError_En(jsonConfig.GetParseError()) + << " at offset " << jsonConfig.GetErrorOffset() << "." << endl; + return -1; + } + + string jsonError; + if (!validateConfigJson(jsonConfig, jsonError)) { + cerr << "Invalid JSON file: " << jsonError << "." << endl; + return -1; + } config.readConfig(jsonConfig, hasClippingBox, clippingBox); } catch (...) { diff --git a/test/config_validator.test.cpp b/test/config_validator.test.cpp new file mode 100644 index 00000000..6bf2116f --- /dev/null +++ b/test/config_validator.test.cpp @@ -0,0 +1,183 @@ +#include +#include +#include + +#include "config_validator.h" +#include "external/minunit.h" +#include "rapidjson/filereadstream.h" + +bool validateString(const char* json, std::string &error) { + rapidjson::Document doc; + doc.Parse(json); + if (doc.HasParseError()) { + error = "parse error"; + return false; + } + return validateConfigJson(doc, error); +} + +bool validateFile(const char* filename, std::string &error) { + FILE* fp = fopen(filename, "r"); + if (!fp) { + error = "could not open file"; + return false; + } + + char readBuffer[65536]; + rapidjson::FileReadStream is(fp, readBuffer, sizeof(readBuffer)); + rapidjson::Document doc; + doc.ParseStream(is); + fclose(fp); + if (doc.HasParseError()) { + error = "parse error"; + return false; + } + + return validateConfigJson(doc, error); +} + +MU_TEST(test_valid_minimal_config) { + std::string error; + mu_check(validateString(R"({ + "settings": { + "basezoom": 14, + "minzoom": 0, + "maxzoom": 14, + "include_ids": false, + "compress": "gzip", + "name": "Test", + "version": "1", + "description": "Test config" + }, + "layers": { + "water": { + "minzoom": 0, + "maxzoom": 14 + } + } + })", error)); + mu_check(error.empty()); +} + +MU_TEST(test_missing_required_setting) { + std::string error; + mu_check(!validateString(R"({ + "settings": { + "basezoom": 14, + "minzoom": 0, + "maxzoom": 14, + "compress": "gzip", + "name": "Test", + "version": "1", + "description": "Test config" + }, + "layers": { + "water": { + "minzoom": 0, + "maxzoom": 14 + } + } + })", error)); + mu_check(error.find("required") != std::string::npos); + mu_check(error.find("\"include_ids\"") != std::string::npos); +} + +MU_TEST(test_missing_multiple_required_settings) { + std::string error; + mu_check(!validateString(R"({ + "settings": { + "basezoom": 14, + "minzoom": 0, + "maxzoom": 14, + "include_ids": true + }, + "layers": { + "transportation": { + "minzoom": 12, + "maxzoom": 14 + } + } + })", error)); + mu_check(error.find("\"compress\"") != std::string::npos); + mu_check(error.find("\"name\"") != std::string::npos); + mu_check(error.find("\"version\"") != std::string::npos); + mu_check(error.find("\"description\"") != std::string::npos); +} + +MU_TEST(test_invalid_layer_type) { + std::string error; + mu_check(!validateString(R"({ + "settings": { + "basezoom": 14, + "minzoom": 0, + "maxzoom": 14, + "include_ids": false, + "compress": "gzip", + "name": "Test", + "version": "1", + "description": "Test config" + }, + "layers": { + "water": { + "minzoom": "0", + "maxzoom": 14 + } + } + })", error)); + mu_check(error.find("type") != std::string::npos); + mu_check(error.find("#/layers/water/minzoom") != std::string::npos); + mu_check(error.find("expected \"integer\", got string") != std::string::npos); +} + +MU_TEST(test_invalid_source_columns) { + std::string error; + mu_check(!validateString(R"({ + "settings": { + "basezoom": 14, + "minzoom": 0, + "maxzoom": 14, + "include_ids": false, + "compress": "gzip", + "name": "Test", + "version": "1", + "description": "Test config" + }, + "layers": { + "water": { + "minzoom": 0, + "maxzoom": 14, + "source_columns": false + } + } + })", error)); + mu_check(error.find("oneOf") != std::string::npos); +} + +MU_TEST(test_bundled_configs) { + std::vector configs = { + "resources/config-coastline.json", + "resources/config-debug.json", + "resources/config-example.json", + "resources/config-openmaptiles.json" + }; + for (const auto &config: configs) { + std::string error; + mu_check(validateFile(config.c_str(), error)); + mu_check(error.empty()); + } +} + +MU_TEST_SUITE(test_suite_config_validator) { + MU_RUN_TEST(test_valid_minimal_config); + MU_RUN_TEST(test_missing_required_setting); + MU_RUN_TEST(test_missing_multiple_required_settings); + MU_RUN_TEST(test_invalid_layer_type); + MU_RUN_TEST(test_invalid_source_columns); + MU_RUN_TEST(test_bundled_configs); +} + +int main() { + MU_RUN_SUITE(test_suite_config_validator); + MU_REPORT(); + return MU_EXIT_CODE; +}