diff --git a/Gemfile b/Gemfile index 5119c17..a78b50e 100644 --- a/Gemfile +++ b/Gemfile @@ -12,6 +12,8 @@ rescue StandardError nil end +gem "lutaml", github: "lutaml/lutaml", branch: "main" +gem "lutaml-model", github: "lutaml/lutaml-model", branch: "fix/xsd-target-namespace-prefix" gem "rake", "~> 13" gem "rspec" gem "rspec-html-matchers" diff --git a/README.adoc b/README.adoc index dc2f205..64a81e5 100644 --- a/README.adoc +++ b/README.adoc @@ -14,6 +14,7 @@ within a Metanorma document: * Enterprise Architect exported UML files in XMI format (`*.xmi`) * LutaML GML Dictionary files (`*.xml`) * JSON or YAML files (`*.json|*.yml|*.yaml`) +* XML Schema files (`*.xsd`) == Installation @@ -34,6 +35,8 @@ link:docs/usages/lutaml-gml.adoc[Usage with LutaML GML Dictionary by lutaml_gml_ link:docs/usages/json_yaml.adoc[Usage with JSON or YAML files by data2text, yaml2text or json2text] +link:docs/usages/lutaml-xsd.adoc[Usage with XML Schema files by lutaml_xsd] + == Documentation Please refer to https://www.metanorma.org. diff --git a/docs/usages/lutaml-xsd.adoc b/docs/usages/lutaml-xsd.adoc new file mode 100644 index 0000000..9a33e35 --- /dev/null +++ b/docs/usages/lutaml-xsd.adoc @@ -0,0 +1,94 @@ +== Usage with LutaML XSD + +=== Overview + +The `lutaml_xsd` macro parses *XML Schema (XSD)* files through `lutaml-model` +and exposes the parsed schema object to *Liquid* templates. + +=== Syntax + +[source,adoc] +----- +[lutaml_xsd,,[, option1=value1, option2=value2, ...]] +---- + +---- +----- + +* ``: Path to the XSD file to be processed. +* ``: The name of the context variable to use in the template. +* `option1=value1, ...`: Optional parameters (<>). + +[[options]] +=== Options + +* `location`: Base URL or path for resolving `` and `` + statements in the XSD. When omitted, the directory of `` is used. + +=== Liquid Template Context + +The context variable (e.g., `unitsml`) exposes the parsed +`Lutaml::Xml::Schema::Xsd::Schema` object through its Liquid drop. + +Commonly used schema collections: + +* `element`: List of elements defined in the XSD. +* `complex_type`: List of complex types defined in the XSD. +* `simple_type`, `attribute`, `attribute_group`, `group`, `import`, and + `include`: Other schema components exposed by `lutaml-model`. + +Commonly used helpers: + +* `elements_sorted_by_name`, `complex_types_sorted_by_name`, + `attribute_groups_sorted_by_name`: Sorted schema collections. +* `used_by`, `child_elements`, `attribute_elements`, and `referenced_type`: + Component helpers exposed by parsed XSD objects. + +=== Example: Listing Elements and Complex Types + +[source,adoc] +----- += Elements +[lutaml_xsd,path/to/unitsml.xsd,unitsml] +---- +{% for element in unitsml.elements_sorted_by_name %} +Name: *{{ element.name }}* +Type: *{{ element.type }}* +Used by: {{ element.used_by | map: "name" | join: ", " }} +{% endfor %} +---- + += ComplexTypes +[lutaml_xsd,path/to/unitsml.xsd,unitsml] +---- +{% for complex_type in unitsml.complex_types_sorted_by_name %} +Name: *{{ complex_type.name }}* +Children: {{ complex_type.child_elements | map: "name" | join: ", " }} +Attributes: {{ complex_type.attribute_elements | map: "name" | join: ", " }} +{% endfor %} +---- +----- + +=== Example: Using with Remote XSD and Options + +[source,adoc] +----- +[lutaml_xsd,path/to/omml.xsd,omml, location=https://raw.githubusercontent.com/t-yuki/ooxml-xsd/refs/heads/master] +---- +{% for element in omml.element %} +Name: *{{ element.name }}* +Type: *{{ element.type }}* +{% endfor %} +---- +----- + +=== Use Cases + +* Generate documentation for XML schemas. +* Extract and list schema elements and types or other details. +* Customize output using Liquid templates. + +=== Notes + +* The macro supports local files at ``. +* You can use all standard *Liquid* template features for formatting and logic. diff --git a/lib/metanorma-plugin-lutaml.rb b/lib/metanorma-plugin-lutaml.rb index b1a3a26..8925065 100644 --- a/lib/metanorma-plugin-lutaml.rb +++ b/lib/metanorma-plugin-lutaml.rb @@ -3,7 +3,9 @@ require "metanorma/plugin/lutaml/json2_text_preprocessor" require "metanorma/plugin/lutaml/yaml2_text_preprocessor" require "metanorma/plugin/lutaml/data2_text_preprocessor" +require "metanorma/plugin/lutaml/base_preprocessor" require "metanorma/plugin/lutaml/lutaml_preprocessor" +require "metanorma/plugin/lutaml/lutaml_xsd_preprocessor" require "metanorma/plugin/lutaml/lutaml_uml_datamodel_description_preprocessor" require "metanorma/plugin/lutaml/lutaml_ea_xmi_preprocessor" require "metanorma/plugin/lutaml/lutaml_xmi_uml_preprocessor" diff --git a/lib/metanorma/plugin/lutaml/base_preprocessor.rb b/lib/metanorma/plugin/lutaml/base_preprocessor.rb new file mode 100644 index 0000000..6fe8cd8 --- /dev/null +++ b/lib/metanorma/plugin/lutaml/base_preprocessor.rb @@ -0,0 +1,289 @@ +# frozen_string_literal: true + +require "liquid" +require "asciidoctor" +require "asciidoctor/reader" +require "lutaml" +require "metanorma/plugin/lutaml/utils" +require "metanorma/plugin/lutaml/asciidoctor/preprocessor" +require "metanorma/plugin/lutaml/express_remarks_decorator" + +module Metanorma + module Plugin + module Lutaml + # Class for processing Lutaml files + class BasePreprocessor < ::Asciidoctor::Extensions::Preprocessor + include Utils + + REMARKS_ATTRIBUTE = "remarks" + FILE_SYSTEM_PATTERNS = [ + "%s.liquid", + "_%s.liquid", + "_%s.adoc", + ].freeze + + def process(document, reader) # rubocop:disable Metrics/MethodLength + r = Asciidoctor::PreprocessorNoIfdefsReader.new(document, + reader.lines) + input_lines = r.readlines.to_enum + + express_indexes = Utils.parse_document_express_indexes( + document, + input_lines, + ) + + result_content = process_input_lines( + document: document, + input_lines: input_lines, + express_indexes: express_indexes, + ) + + Asciidoctor::PreprocessorNoIfdefsReader.new(document, result_content) + end + + protected + + def load_lutaml_file(document, file_path, options) + File.open( + Utils.relative_file_path(document, file_path), + encoding: "UTF-8", + ) do |file| + ::Lutaml::Parser.parse(file, options: options) + end + end + + def index_type_name + "EXPRESS" + end + + def index_missing_message(path) + "Unable to load #{index_type_name} index for `#{path}`, " \ + "please define it at `:lutaml-express-index:` or specify " \ + "the full path." + end + + private + + def process_input_lines(document:, input_lines:, express_indexes:) + result = [] + loop do + result.push( + *process_text_blocks(document, input_lines, express_indexes), + ) + end + result + end + + def process_text_blocks(document, input_lines, express_indexes) # rubocop:disable Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/MethodLength + line = input_lines.next + block_header_match = lutaml_liquid?(line) + + return [line] if block_header_match.nil? + + index_names = block_header_match[:index_names].split(";").map(&:strip) + context_name = block_header_match[:context_name].strip + + options = block_header_match[:options] && + parse_options(block_header_match[:options].to_s.strip) || {} + + end_mark = input_lines.next + + render_liquid_template( + document: document, + lines: extract_block_lines(input_lines, end_mark), + index_names: index_names, + context_name: context_name, + options: options, + indexes: express_indexes, + ) + end + + def extract_block_lines(input_lines, end_mark) + block = [] + while (block_line = input_lines.next) != end_mark + block.push(block_line) + end + block + end + + def gather_context_liquid_items( # rubocop:disable Metrics/AbcSize,Metrics/MethodLength,Metrics/ParameterLists + index_names:, document:, indexes:, options: {} + ) + index_names.map do |path| # rubocop:disable Metrics/BlockLength + if indexes[path] && indexes[path][:model] + repo = indexes[path][:model] + repo = update_repo(options, repo) + indexes[path][:liquid_drop] ||= repo.to_liquid + else + full_path = Utils.relative_file_path(document, path) + unless File.file?(full_path) + raise StandardError.new(index_missing_message(path)) + end + repo = load_lutaml_file(document, path, options) + repo = update_repo(options, repo) + indexes[path] = { + liquid_drop: repo.to_liquid, + } + end + + indexes[path] + end + end + + def update_repo(options, repo) + # Unwrap repo if it's a cache + repo = repo.content if repo.is_a? Expressir::Model::Cache + + # Process each schema + repo.schemas.each do |schema| + options["relative_path_prefix"] = + relative_path_prefix(options, schema) + update_remarks(schema, options) + end + + repo + end + + def update_remarks(model, options) + model.remarks = decorate_remarks(options, model.remarks) + model.remark_items&.each do |ri| + ri.remarks = decorate_remarks(options, ri.remarks) + end + + model.children.each do |child| + if child.respond_to?(:remarks) && child.respond_to?(:remark_items) + update_remarks(child, options) + end + end + end + + def relative_path_prefix(options, model) + return nil if options.nil? || options["document"].nil? + + document = options["document"] + file_path = File.dirname(model.file) + docfile_directory = File.dirname( + document.attributes["docfile"] || ".", + ) + document + .path_resolver + .system_path(file_path, docfile_directory) + end + + def decorate_remarks(options, remarks) + return [] unless remarks + + remarks.map do |remark| + ::Metanorma::Plugin::Lutaml::ExpressRemarksDecorator + .call(remark, options) + end + end + + def read_config_yaml_file(document, file_path) # rubocop:disable Metrics/MethodLength + return {} if file_path.nil? + + relative_file_path = Utils.relative_file_path(document, file_path) + config_yaml = YAML.safe_load( + File.read(relative_file_path, encoding: "UTF-8"), + ) + + return {} unless config_yaml["schemas"] + + unless config_yaml["schemas"].is_a?(Hash) + raise StandardError.new( + "[lutaml_express_liquid] attribute `config_yaml` must point " \ + "to a YAML file that has the `schemas` key containing a hash.", + ) + end + + { "selected_schemas" => config_yaml["schemas"].keys } + end + + def render_liquid_template(document:, lines:, context_name:, # rubocop:disable Metrics/AbcSize,Metrics/MethodLength,Metrics/ParameterLists + index_names:, options:, indexes:) + # Process options and configuration + options = process_options(document, options) + + # Get all context items in one go + all_items = gather_context_liquid_items( + index_names: index_names, document: document, indexes: indexes, + options: options.merge("document" => document) + ) + + file_system = file_system(options, document) + + # Parse template once outside the loop + template = template(lines) + template.registers[:file_system] = file_system + + # Render for each item + all_items.map do |item| + template.assigns[context_name] = item[:liquid_drop] + template.assigns["ordered_schemas"] = reorder_schemas( + item[:liquid_drop], options + ) + template.assigns["schemas_order"] = options["selected_schemas"] + assign_options_in_liquid(template, options) + template.render + end.flatten + rescue StandardError => e + ::Metanorma::Util + .log("[LutamlPreprocessor] Failed to parse LutaML block: " \ + "#{e.message}", :error) + raise e + end + + def file_system(options, document) + # Setup include paths for liquid templates + include_paths = [Utils.relative_file_path(document, "")] + options["include_path"]&.split(",")&.each do |path| + # resolve include_path relative to the document + include_paths.push(Utils.relative_file_path(document, path)) + end + ::Metanorma::Plugin::Lutaml::Liquid::LocalFileSystem + .new(include_paths, FILE_SYSTEM_PATTERNS) + end + + def template(lines) + ::Liquid::Template.parse( + lines.join("\n"), + environment: create_liquid_environment, + ) + end + + def reorder_schemas(repo_liquid, options) + return repo_liquid.schemas unless options["selected_schemas"] + + ordered_schemas = [] + options["selected_schemas"].each do |schema_name| + ordered_schema = repo_liquid.schemas.find do |schema| + schema.id == schema_name || schema.file_basename == schema_name + end + ordered_schemas.push(ordered_schema) + end + + ordered_schemas + end + + def process_options(document, options) + # Process config file if specified + if (config_yaml_path = options.delete("config_yaml")) + config = read_config_yaml_file(document, config_yaml_path) + if config["selected_schemas"] + options["selected_schemas"] = + config["selected_schemas"] + end + end + options + end + + def parse_options(options_string) + options_string + .to_s + .scan(/,\s*([^=]+?)=(\s*[^,]+)/) + .to_h { |elem| elem.map(&:strip) } + end + end + end + end +end diff --git a/lib/metanorma/plugin/lutaml/lutaml_ea_xmi_base.rb b/lib/metanorma/plugin/lutaml/lutaml_ea_xmi_base.rb index 0f65062..58d028b 100644 --- a/lib/metanorma/plugin/lutaml/lutaml_ea_xmi_base.rb +++ b/lib/metanorma/plugin/lutaml/lutaml_ea_xmi_base.rb @@ -14,10 +14,6 @@ module Lutaml module LutamlEaXmiBase include Utils - LIQUID_INCLUDE_PATH = File.join( - Gem.loaded_specs["metanorma-plugin-lutaml"].full_gem_path, - "lib", "metanorma", "plugin", "lutaml", "liquid_templates" - ) DEFAULT_RENDER_INCLUDE = "packages" RENDER_STYLES_INCLUDES = { "default" => "packages", @@ -422,7 +418,7 @@ def model_representation(lutaml_doc, document, add_context, options) # rubocop:d end def template_path(document, template_path) - return LIQUID_INCLUDE_PATH if template_path.nil? + return Utils::LIQUID_INCLUDE_PATH if template_path.nil? Utils.relative_file_path(document, template_path) end diff --git a/lib/metanorma/plugin/lutaml/lutaml_preprocessor.rb b/lib/metanorma/plugin/lutaml/lutaml_preprocessor.rb index 70ab6a1..7ba9b6c 100644 --- a/lib/metanorma/plugin/lutaml/lutaml_preprocessor.rb +++ b/lib/metanorma/plugin/lutaml/lutaml_preprocessor.rb @@ -1,21 +1,10 @@ # frozen_string_literal: true -require "liquid" -require "asciidoctor" -require "asciidoctor/reader" -require "lutaml" -require "metanorma/plugin/lutaml/utils" -require "metanorma/plugin/lutaml/asciidoctor/preprocessor" -require "metanorma/plugin/lutaml/express_remarks_decorator" - module Metanorma module Plugin module Lutaml # Class for processing Lutaml files - class LutamlPreprocessor < ::Asciidoctor::Extensions::Preprocessor - include Utils - - REMARKS_ATTRIBUTE = "remarks" + class LutamlPreprocessor < BasePreprocessor EXPRESS_PREPROCESSOR_REGEX = %r{ ^ # Start of line \[ # Opening bracket @@ -28,26 +17,7 @@ class LutamlPreprocessor < ::Asciidoctor::Extensions::Preprocessor (?[^,]+)? # Optional context name (?,.*)? # Optional options \] # Closing bracket - }x - - def process(document, reader) # rubocop:disable Metrics/MethodLength - r = Asciidoctor::PreprocessorNoIfdefsReader.new(document, - reader.lines) - input_lines = r.readlines.to_enum - - express_indexes = Utils.parse_document_express_indexes( - document, - input_lines, - ) - - result_content = process_input_lines( - document: document, - input_lines: input_lines, - express_indexes: express_indexes, - ) - - Asciidoctor::PreprocessorNoIfdefsReader.new(document, result_content) - end + }x.freeze protected @@ -55,229 +25,8 @@ def lutaml_liquid?(line) line.match(EXPRESS_PREPROCESSOR_REGEX) end - def load_express_lutaml_file(document, file_path) - ::Lutaml::Parser.parse( - File.new( - Utils.relative_file_path(document, file_path), - encoding: "UTF-8", - ), - ) - end - - private - - def process_input_lines(document:, input_lines:, express_indexes:) - result = [] - loop do - result.push( - *process_text_blocks(document, input_lines, express_indexes), - ) - end - result - end - - def process_text_blocks(document, input_lines, express_indexes) # rubocop:disable Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/MethodLength - line = input_lines.next - block_header_match = lutaml_liquid?(line) - - return [line] if block_header_match.nil? - - index_names = block_header_match[:index_names].split(";").map(&:strip) - context_name = block_header_match[:context_name].strip - - options = (block_header_match[:options] && - parse_options(block_header_match[:options].to_s.strip)) || {} - - end_mark = input_lines.next - - render_liquid_template( - document: document, - lines: extract_block_lines(input_lines, end_mark), - index_names: index_names, - context_name: context_name, - options: options, - indexes: express_indexes, - ) - end - - def extract_block_lines(input_lines, end_mark) - block = [] - while (block_line = input_lines.next) != end_mark - block.push(block_line) - end - block - end - - def gather_context_liquid_items( # rubocop:disable Metrics/AbcSize,Metrics/MethodLength,Metrics/ParameterLists - index_names:, document:, indexes:, options: {} - ) - index_names.map do |path| # rubocop:disable Metrics/BlockLength - if indexes[path] && indexes[path][:model] - repo = indexes[path][:model] - repo = update_repo(options, repo) - indexes[path][:liquid_drop] ||= repo.to_liquid - else - full_path = Utils.relative_file_path(document, path) - unless File.file?(full_path) - raise StandardError.new( - "Unable to load EXPRESS index for `#{path}`, " \ - "please define it at `:lutaml-express-index:` or specify " \ - "the full path.", - ) - end - repo = load_express_lutaml_file(document, path) - repo = update_repo(options, repo) - indexes[path] = { - liquid_drop: repo.to_liquid, - } - end - - indexes[path] - end - end - - def update_repo(options, repo) - # Unwrap repo if it's a cache - repo = repo.content if repo.is_a? Expressir::Model::Cache - - # Process each schema - repo.schemas.each do |schema| - options["relative_path_prefix"] = - relative_path_prefix(options, schema) - update_remarks(schema, options) - end - - repo - end - - def update_remarks(model, options) - model.remarks = decorate_remarks(options, model.remarks) - model.remark_items&.each do |ri| - ri.remarks = decorate_remarks(options, ri.remarks) - end - - model.children.each do |child| - if child.respond_to?(:remarks) && child.respond_to?(:remark_items) - update_remarks(child, options) - end - end - end - - def relative_path_prefix(options, model) - return nil if options.nil? || options["document"].nil? - - document = options["document"] - file_path = File.dirname(model.file) - docfile_directory = File.dirname( - document.attributes["docfile"] || ".", - ) - document - .path_resolver - .system_path(file_path, docfile_directory) - end - - def decorate_remarks(options, remarks) - return [] unless remarks - - remarks.map do |remark| - ::Metanorma::Plugin::Lutaml::ExpressRemarksDecorator - .call(remark, options) - end - end - - def read_config_yaml_file(document, file_path) # rubocop:disable Metrics/MethodLength - return {} if file_path.nil? - - relative_file_path = Utils.relative_file_path(document, file_path) - config_yaml = YAML.safe_load( - File.read(relative_file_path, encoding: "UTF-8"), - ) - - return {} unless config_yaml["schemas"] - - unless config_yaml["schemas"].is_a?(Hash) - raise StandardError.new( - "[lutaml_express_liquid] attribute `config_yaml` must point " \ - "to a YAML file that has the `schemas` key containing a hash.", - ) - end - - { "selected_schemas" => config_yaml["schemas"].keys } - end - - def render_liquid_template(document:, lines:, context_name:, # rubocop:disable Metrics/AbcSize,Metrics/MethodLength,Metrics/ParameterLists - index_names:, options:, indexes:) - # Process options and configuration - options = process_options(document, options) - - # Get all context items in one go - all_items = gather_context_liquid_items( - index_names: index_names, document: document, indexes: indexes, - options: options.merge("document" => document) - ) - - # Setup include paths for liquid templates - include_paths = [Utils.relative_file_path(document, "")] - options["include_path"]&.split(",")&.each do |path| - # resolve include_path relative to the document - include_paths.push(Utils.relative_file_path(document, path)) - end - - file_system = ::Metanorma::Plugin::Lutaml::Liquid::LocalFileSystem - .new(include_paths, ["%s.liquid", "_%s.liquid", "_%s.adoc"]) - - # Parse template once outside the loop - template = ::Liquid::Template - .parse(lines.join("\n"), environment: create_liquid_environment) - template.registers[:file_system] = file_system - - # Render for each item - all_items.map do |item| - template.assigns[context_name] = item[:liquid_drop] - template.assigns["ordered_schemas"] = reorder_schemas( - item[:liquid_drop], options - ) - template.assigns["schemas_order"] = options["selected_schemas"] - template.render - end.flatten - rescue StandardError => e - ::Metanorma::Util - .log("[LutamlPreprocessor] Failed to parse LutaML block: " \ - "#{e.message}", :error) - raise e - end - - def reorder_schemas(repo_liquid, options) - return repo_liquid.schemas unless options["selected_schemas"] - - ordered_schemas = [] - options["selected_schemas"].each do |schema_name| - ordered_schema = repo_liquid.schemas.find do |schema| - schema.id == schema_name || schema.file_basename == schema_name - end - ordered_schemas.push(ordered_schema) - end - - ordered_schemas - end - - def process_options(document, options) - # Process config file if specified - if (config_yaml_path = options.delete("config_yaml")) - config = read_config_yaml_file(document, config_yaml_path) - if config["selected_schemas"] - options["selected_schemas"] = - config["selected_schemas"] - end - end - options - end - - def parse_options(options_string) - options_string - .to_s - .scan(/,\s*([^=]+?)=(\s*[^,]+)/) - .to_h { |elem| elem.map(&:strip) } + def assign_options_in_liquid(template, _options = {}) + template end end end diff --git a/lib/metanorma/plugin/lutaml/lutaml_xsd_preprocessor.rb b/lib/metanorma/plugin/lutaml/lutaml_xsd_preprocessor.rb new file mode 100644 index 0000000..72e45e3 --- /dev/null +++ b/lib/metanorma/plugin/lutaml/lutaml_xsd_preprocessor.rb @@ -0,0 +1,165 @@ +# frozen_string_literal: true + +require "lutaml/xml/parsers/xsd" + +module Metanorma + module Plugin + module Lutaml + class LutamlXsdPreprocessor < BasePreprocessor + XSD_PREPROCESSOR_REGEX = %r{ + ^ # Start of line + \[ # Opening bracket + (?:\blutaml_xsd\b) # lutaml_xsd + , # Comma separator + (?[^,]+)? # Optional index names + ,? # Optional comma + (?[^,]+)? # Optional context name + (?,.*)? # Optional options + \] # Closing bracket + }x.freeze + XSD_LIQUID_BLOCK_REGEX = %r{ + ^ # Start of line + (?:\blutaml_xsd\b):: # lutaml_xsd + (?[^\[]+) # Index names + \[ # Opening bracket + (?[^,]+) # Context name + , # Comma separator + (?