From 9c9399f533d79c68f2368aa8774d5cac4a4fdcac Mon Sep 17 00:00:00 2001 From: Carbo Date: Mon, 13 Jul 2026 17:07:38 -0700 Subject: [PATCH] Add sentencepiece@0.2.2 --- modules/sentencepiece/0.2.2/MODULE.bazel | 26 + .../sentencepiece/0.2.2/overlay/BUILD.bazel | 31 + .../0.2.2/overlay/bcr_tests/BUILD.bazel | 11 + .../0.2.2/overlay/bcr_tests/MODULE.bazel | 10 + .../0.2.2/overlay/bcr_tests/roundtrip_test.cc | 66 ++ .../0.2.2/overlay/src/BUILD.bazel | 263 ++++++ .../patches/use_bcr_dependency_includes.patch | 839 ++++++++++++++++++ modules/sentencepiece/0.2.2/presubmit.yml | 92 ++ modules/sentencepiece/0.2.2/source.json | 16 + modules/sentencepiece/metadata.json | 3 +- 10 files changed, 1356 insertions(+), 1 deletion(-) create mode 100644 modules/sentencepiece/0.2.2/MODULE.bazel create mode 100644 modules/sentencepiece/0.2.2/overlay/BUILD.bazel create mode 100644 modules/sentencepiece/0.2.2/overlay/bcr_tests/BUILD.bazel create mode 100644 modules/sentencepiece/0.2.2/overlay/bcr_tests/MODULE.bazel create mode 100644 modules/sentencepiece/0.2.2/overlay/bcr_tests/roundtrip_test.cc create mode 100644 modules/sentencepiece/0.2.2/overlay/src/BUILD.bazel create mode 100644 modules/sentencepiece/0.2.2/patches/use_bcr_dependency_includes.patch create mode 100644 modules/sentencepiece/0.2.2/presubmit.yml create mode 100644 modules/sentencepiece/0.2.2/source.json diff --git a/modules/sentencepiece/0.2.2/MODULE.bazel b/modules/sentencepiece/0.2.2/MODULE.bazel new file mode 100644 index 00000000000..65bc1d41ee7 --- /dev/null +++ b/modules/sentencepiece/0.2.2/MODULE.bazel @@ -0,0 +1,26 @@ +"""SentencePiece is an unsupervised text tokenizer and detokenizer mainly for Neural Network-based text generation systems.""" + +module( + name = "sentencepiece", + version = "0.2.2", + bazel_compatibility = [">=7.2.1"], +) + +# Keep in sync with the abseil-cpp GIT_TAG in CMakeLists.txt. +bazel_dep(name = "abseil-cpp", version = "20260526.0") + +# Upstream darts-clone v0.32h plus the compatibility helpers (copy_array, +# validate, ...) that the vendored third_party/darts_clone copy carries. +bazel_dep(name = "darts-clone", version = "0.32h.bcr.1") + +# Upstream esaxx at the commit the vendored third_party/esaxx copy is based +# on. The .bcr.1 revision exports esa.hxx and carries the index_type(0) +# template-deduction fix that the int64_t instantiation of esaxx() needs. +bazel_dep(name = "esaxx", version = "20250106.1.bcr.1") +bazel_dep(name = "platforms", version = "1.1.0") + +# The .pb.cc/.pb.h files are regenerated from src/*.proto at build time, so +# the Bazel build follows the SPM_PROTOBUF_PROVIDER=package CMake +# configuration instead of the vendored third_party/protobuf-lite runtime. +bazel_dep(name = "protobuf", version = "35.1") +bazel_dep(name = "rules_cc", version = "0.2.18") diff --git a/modules/sentencepiece/0.2.2/overlay/BUILD.bazel b/modules/sentencepiece/0.2.2/overlay/BUILD.bazel new file mode 100644 index 00000000000..14826b1455b --- /dev/null +++ b/modules/sentencepiece/0.2.2/overlay/BUILD.bazel @@ -0,0 +1,31 @@ +# Copyright 2018 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License.! + +package(default_visibility = ["//visibility:public"]) + +exports_files([ + "VERSION.txt", + "config.h.in", + "LICENSE", +]) + +alias( + name = "sentencepiece", + actual = "//src:sentencepiece", +) + +alias( + name = "sentencepiece_train", + actual = "//src:sentencepiece_train", +) diff --git a/modules/sentencepiece/0.2.2/overlay/bcr_tests/BUILD.bazel b/modules/sentencepiece/0.2.2/overlay/bcr_tests/BUILD.bazel new file mode 100644 index 00000000000..55e2c9c3a28 --- /dev/null +++ b/modules/sentencepiece/0.2.2/overlay/bcr_tests/BUILD.bazel @@ -0,0 +1,11 @@ +load("@rules_cc//cc:defs.bzl", "cc_test") + +cc_test( + name = "roundtrip_test", + srcs = ["roundtrip_test.cc"], + deps = [ + "@abseil-cpp//absl/status", + "@sentencepiece//:sentencepiece", + "@sentencepiece//:sentencepiece_train", + ], +) diff --git a/modules/sentencepiece/0.2.2/overlay/bcr_tests/MODULE.bazel b/modules/sentencepiece/0.2.2/overlay/bcr_tests/MODULE.bazel new file mode 100644 index 00000000000..d7e4ad4b05b --- /dev/null +++ b/modules/sentencepiece/0.2.2/overlay/bcr_tests/MODULE.bazel @@ -0,0 +1,10 @@ +module(name = "sentencepiece_bcr_tests") + +bazel_dep(name = "sentencepiece") +local_path_override( + module_name = "sentencepiece", + path = "..", +) + +bazel_dep(name = "abseil-cpp", version = "20260526.0") +bazel_dep(name = "rules_cc", version = "0.2.18") diff --git a/modules/sentencepiece/0.2.2/overlay/bcr_tests/roundtrip_test.cc b/modules/sentencepiece/0.2.2/overlay/bcr_tests/roundtrip_test.cc new file mode 100644 index 00000000000..1a7eed19fb9 --- /dev/null +++ b/modules/sentencepiece/0.2.2/overlay/bcr_tests/roundtrip_test.cc @@ -0,0 +1,66 @@ +// Trains a small model fully in memory, then checks that encoding and +// decoding roundtrips. This replaces the upstream //src:spm_test, whose +// test data (data/) is not shipped in the release tarball. + +#include +#include +#include + +#include "absl/status/status.h" +#include "sentencepiece_processor.h" +#include "sentencepiece_trainer.h" + +#define CHECK_TEST(condition) \ + if (!(condition)) { \ + std::cerr << __FILE__ << ":" << __LINE__ << " check failed: " \ + << #condition << std::endl; \ + return 1; \ + } + +#define CHECK_OK_TEST(expr) \ + { \ + const absl::Status _status = (expr); \ + if (!_status.ok()) { \ + std::cerr << __FILE__ << ":" << __LINE__ << " status not ok: " \ + << _status << std::endl; \ + return 1; \ + } \ + } + +int main() { + const std::vector base = { + "The quick brown fox jumps over the lazy dog.", + "SentencePiece is an unsupervised text tokenizer and detokenizer.", + "Hello world. Hello Bazel.", + "I saw a girl with a telescope.", + }; + std::vector sentences; + for (int i = 0; i < 200; ++i) { + sentences.push_back(base[i % base.size()]); + } + + std::string serialized_model; + CHECK_OK_TEST(sentencepiece::SentencePieceTrainer::Train( + "--vocab_size=60 --hard_vocab_limit=false --minloglevel=1", sentences, + &serialized_model)); + CHECK_TEST(!serialized_model.empty()); + + sentencepiece::SentencePieceProcessor sp; + CHECK_OK_TEST(sp.LoadFromSerializedProto(serialized_model)); + + const std::string input = "Hello world."; + std::vector pieces; + CHECK_OK_TEST(sp.Encode(input, &pieces)); + CHECK_TEST(!pieces.empty()); + + std::string detokenized; + CHECK_OK_TEST(sp.Decode(pieces, &detokenized)); + CHECK_TEST(detokenized == input); + + std::vector ids; + CHECK_OK_TEST(sp.Encode(input, &ids)); + CHECK_TEST(!ids.empty()); + + std::cout << "PASS" << std::endl; + return 0; +} diff --git a/modules/sentencepiece/0.2.2/overlay/src/BUILD.bazel b/modules/sentencepiece/0.2.2/overlay/src/BUILD.bazel new file mode 100644 index 00000000000..18b961b6971 --- /dev/null +++ b/modules/sentencepiece/0.2.2/overlay/src/BUILD.bazel @@ -0,0 +1,263 @@ +# Copyright 2018 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License.! + +load("@protobuf//bazel:cc_proto_library.bzl", "cc_proto_library") +load("@protobuf//bazel:proto_library.bzl", "proto_library") +load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library") + +package(default_visibility = ["//visibility:public"]) + +NOT_WIN_COPTS = [ + "-pthread", + "-Wall", +] + +# Generates config.h from config.h.in, like the configure_file() step in +# CMakeLists.txt. The version is read from VERSION.txt. +genrule( + name = "config_h", + srcs = [ + "//:VERSION.txt", + "//:config.h.in", + ], + outs = ["config.h"], + cmd = "VERSION=$$(tr -d ' \\r\\n' < $(location //:VERSION.txt)); " + + "sed -e \"s/@PROJECT_VERSION@/$${VERSION}/g\" " + + "-e \"s/@PROJECT_NAME@/sentencepiece/g\" " + + "-e \"s|@INSTALL_DATADIR@|share/sentencepiece|g\" " + + "$(location //:config.h.in) > $@", +) + +cc_library( + name = "config", + hdrs = [":config_h"], + includes = ["."], +) + +# The protobuf code is regenerated from the .proto files at build time with +# the protobuf module from the Bazel Central Registry, matching the +# SPM_PROTOBUF_PROVIDER=package CMake configuration. The pre-generated +# sources in builtin_pb/ and the vendored third_party/protobuf-lite runtime +# are only used by CMake (SPM_PROTOBUF_PROVIDER=internal, its default). +proto_library( + name = "sentencepiece_proto", + srcs = ["sentencepiece.proto"], +) + +cc_proto_library( + name = "sentencepiece_cc_proto", + deps = [":sentencepiece_proto"], +) + +proto_library( + name = "sentencepiece_model_proto", + srcs = ["sentencepiece_model.proto"], +) + +cc_proto_library( + name = "sentencepiece_model_cc_proto", + deps = [":sentencepiece_model_proto"], +) + +# Wrapper that puts the generated headers on the include path expected by +# the sources (#include "sentencepiece.pb.h") and selects the external +# protobuf branch of the #ifdef _USE_EXTERNAL_PROTOBUF includes. +cc_library( + name = "pb", + defines = ["_USE_EXTERNAL_PROTOBUF"], + includes = ["."], + deps = [ + ":sentencepiece_cc_proto", + ":sentencepiece_model_cc_proto", + ], +) + +# Runtime library (the `sentencepiece` target in CMake; SPM_SRCS). +cc_library( + name = "sentencepiece", + srcs = [ + "bpe_model.cc", + "char_model.cc", + "filesystem.cc", + "init.cc", + "model_factory.cc", + "model_interface.cc", + "normalizer.cc", + "sentencepiece_processor.cc", + "unigram_model.cc", + "util.cc", + "word_model.cc", + ], + hdrs = [ + "bpe_model.h", + "char_model.h", + "common.h", + "filesystem.h", + "freelist.h", + "init.h", + "model_factory.h", + "model_interface.h", + "normalizer.h", + "sentencepiece_processor.h", + "unigram_model.h", + "util.h", + "word_model.h", + ], + copts = select({ + "@platforms//os:windows": [], + "//conditions:default": NOT_WIN_COPTS, + }), + linkopts = select({ + "@platforms//cpu:riscv64": ["-latomic"], + "//conditions:default": [], + }), + # @abseil-cpp//absl/base:endian (for base/internal/endian.h, used by + # util.h) is intentionally absent: it is not publicly visible. Its header + # still reaches dependents through the transitive headers of + # @abseil-cpp//absl/strings. + deps = [ + ":config", + ":pb", + "@abseil-cpp//absl/base:core_headers", + "@abseil-cpp//absl/cleanup", + "@abseil-cpp//absl/container:fixed_array", + "@abseil-cpp//absl/container:flat_hash_map", + "@abseil-cpp//absl/container:flat_hash_set", + "@abseil-cpp//absl/flags:config", + "@abseil-cpp//absl/flags:flag", + "@abseil-cpp//absl/flags:parse", + "@abseil-cpp//absl/flags:usage", + "@abseil-cpp//absl/functional:any_invocable", + "@abseil-cpp//absl/functional:function_ref", + "@abseil-cpp//absl/log", + "@abseil-cpp//absl/log:check", + "@abseil-cpp//absl/log:globals", + "@abseil-cpp//absl/log:initialize", + "@abseil-cpp//absl/numeric:bits", + "@abseil-cpp//absl/random", + "@abseil-cpp//absl/status", + "@abseil-cpp//absl/status:status_builder", + "@abseil-cpp//absl/status:status_macros", + "@abseil-cpp//absl/strings", + "@abseil-cpp//absl/strings:str_format", + "@abseil-cpp//absl/synchronization", + "@abseil-cpp//absl/time", + "@abseil-cpp//absl/types:span", + "@darts-clone//:darts-clone", + ], +) + +# Trainer library (the `sentencepiece_train` target in CMake; SPM_TRAIN_SRCS). +cc_library( + name = "sentencepiece_train", + srcs = [ + "bpe_model_trainer.cc", + "builder.cc", + "char_model_trainer.cc", + "pretokenizer_for_training.cc", + "sentencepiece_trainer.cc", + "trainer_factory.cc", + "trainer_interface.cc", + "unicode_script.cc", + "unigram_model_trainer.cc", + "word_model_trainer.cc", + ], + hdrs = [ + "bpe_model_trainer.h", + "builder.h", + "char_model_trainer.h", + "normalization_rule.h", + "pretokenizer_for_training.h", + "sentencepiece_trainer.h", + "spec_parser.h", + "trainer_factory.h", + "trainer_interface.h", + "unicode_script.h", + "unicode_script_map.h", + "unigram_model_trainer.h", + "word_model_trainer.h", + ], + copts = select({ + "@platforms//os:windows": [], + "//conditions:default": NOT_WIN_COPTS, + }), + deps = [ + ":sentencepiece", + "@abseil-cpp//absl/container:btree", + "@abseil-cpp//absl/container:flat_hash_map", + "@abseil-cpp//absl/container:flat_hash_set", + "@abseil-cpp//absl/flags:flag", + "@abseil-cpp//absl/hash", + "@abseil-cpp//absl/random", + "@abseil-cpp//absl/status", + "@abseil-cpp//absl/status:status_builder", + "@abseil-cpp//absl/strings", + "@abseil-cpp//absl/strings:str_format", + "@abseil-cpp//absl/types:span", + "@esaxx//:esa", + ], +) + +cc_binary( + name = "spm_encode", + srcs = ["spm_encode_main.cc"], + deps = [ + ":sentencepiece", + # For trainer_interface.h. + ":sentencepiece_train", + "@abseil-cpp//absl/container:flat_hash_map", + "@abseil-cpp//absl/flags:flag", + "@abseil-cpp//absl/strings", + ], +) + +cc_binary( + name = "spm_decode", + srcs = ["spm_decode_main.cc"], + deps = [ + ":sentencepiece", + "@abseil-cpp//absl/flags:flag", + "@abseil-cpp//absl/strings", + ], +) + +cc_binary( + name = "spm_normalize", + srcs = ["spm_normalize_main.cc"], + deps = [ + ":sentencepiece", + ":sentencepiece_train", + "@abseil-cpp//absl/flags:flag", + ], +) + +cc_binary( + name = "spm_train", + srcs = ["spm_train_main.cc"], + deps = [ + ":sentencepiece", + ":sentencepiece_train", + "@abseil-cpp//absl/flags:flag", + "@abseil-cpp//absl/strings", + ], +) + +cc_binary( + name = "spm_export_vocab", + srcs = ["spm_export_vocab_main.cc"], + deps = [ + ":sentencepiece", + "@abseil-cpp//absl/flags:flag", + ], +) diff --git a/modules/sentencepiece/0.2.2/patches/use_bcr_dependency_includes.patch b/modules/sentencepiece/0.2.2/patches/use_bcr_dependency_includes.patch new file mode 100644 index 00000000000..560b16fba64 --- /dev/null +++ b/modules/sentencepiece/0.2.2/patches/use_bcr_dependency_includes.patch @@ -0,0 +1,839 @@ +diff --git a/src/bpe_model.cc b/src/bpe_model.cc +index e0c4004..f28894d 100644 +--- a/src/bpe_model.cc ++++ b/src/bpe_model.cc +@@ -25,10 +25,10 @@ + #include "freelist.h" + #include "model_interface.h" + #include "sentencepiece_model.pb.h" +-#include "third_party/absl/base/attributes.h" +-#include "third_party/absl/container/flat_hash_map.h" +-#include "third_party/absl/random/random.h" +-#include "third_party/absl/strings/string_view.h" ++#include "absl/base/attributes.h" ++#include "absl/container/flat_hash_map.h" ++#include "absl/random/random.h" ++#include "absl/strings/string_view.h" + #include "util.h" + + namespace sentencepiece { +diff --git a/src/bpe_model.h b/src/bpe_model.h +index 11df76c..2385548 100644 +--- a/src/bpe_model.h ++++ b/src/bpe_model.h +@@ -17,7 +17,7 @@ + + #include "model_interface.h" + #include "sentencepiece_model.pb.h" +-#include "third_party/absl/strings/string_view.h" ++#include "absl/strings/string_view.h" + + namespace sentencepiece { + namespace bpe { +diff --git a/src/bpe_model_trainer.cc b/src/bpe_model_trainer.cc +index 57715d0..3594a66 100644 +--- a/src/bpe_model_trainer.cc ++++ b/src/bpe_model_trainer.cc +@@ -20,13 +20,13 @@ + #include + + #include "pretokenizer_for_training.h" +-#include "third_party/absl/container/flat_hash_set.h" +-#include "third_party/absl/flags/flag.h" +-#include "third_party/absl/hash/hash.h" +-#include "third_party/absl/status/status.h" +-#include "third_party/absl/strings/str_join.h" +-#include "third_party/absl/strings/str_replace.h" +-#include "third_party/absl/strings/string_view.h" ++#include "absl/container/flat_hash_set.h" ++#include "absl/flags/flag.h" ++#include "absl/hash/hash.h" ++#include "absl/status/status.h" ++#include "absl/strings/str_join.h" ++#include "absl/strings/str_replace.h" ++#include "absl/strings/string_view.h" + #include "util.h" + + #ifdef SPM_NLCODEC_BPE +diff --git a/src/bpe_model_trainer.h b/src/bpe_model_trainer.h +index f01edbb..b5ca82d 100644 +--- a/src/bpe_model_trainer.h ++++ b/src/bpe_model_trainer.h +@@ -23,9 +23,9 @@ + #include + + #include "sentencepiece_model.pb.h" +-#include "third_party/absl/container/btree_set.h" +-#include "third_party/absl/container/flat_hash_map.h" +-#include "third_party/absl/status/status.h" ++#include "absl/container/btree_set.h" ++#include "absl/container/flat_hash_map.h" ++#include "absl/status/status.h" + #include "trainer_interface.h" + + namespace sentencepiece::bpe { +diff --git a/src/builder.cc b/src/builder.cc +index e47bd9f..7f37aa4 100644 +--- a/src/builder.cc ++++ b/src/builder.cc +@@ -19,15 +19,15 @@ + #include + + #include "filesystem.h" +-#include "third_party/absl/container/flat_hash_map.h" +-#include "third_party/absl/container/flat_hash_set.h" +-#include "third_party/absl/status/status.h" +-#include "third_party/absl/strings/str_cat.h" +-#include "third_party/absl/strings/str_join.h" +-#include "third_party/absl/strings/str_replace.h" +-#include "third_party/absl/strings/str_split.h" +-#include "third_party/absl/strings/string_view.h" +-#include "third_party/absl/strings/strip.h" ++#include "absl/container/flat_hash_map.h" ++#include "absl/container/flat_hash_set.h" ++#include "absl/status/status.h" ++#include "absl/strings/str_cat.h" ++#include "absl/strings/str_join.h" ++#include "absl/strings/str_replace.h" ++#include "absl/strings/str_split.h" ++#include "absl/strings/string_view.h" ++#include "absl/strings/strip.h" + + #ifdef ENABLE_NFKC_COMPILE + #include +@@ -41,7 +41,7 @@ + #include + + #include "normalizer.h" +-#include "third_party/darts_clone/darts.h" ++#include "darts.h" + #include "util.h" + + #ifndef DISABLE_EMBEDDED_DATA +diff --git a/src/builder.h b/src/builder.h +index 9206f09..8959e3e 100644 +--- a/src/builder.h ++++ b/src/builder.h +@@ -22,8 +22,8 @@ + #include "common.h" + #include "sentencepiece_model.pb.h" + #include "sentencepiece_processor.h" +-#include "third_party/absl/status/status.h" +-#include "third_party/absl/strings/string_view.h" ++#include "absl/status/status.h" ++#include "absl/strings/string_view.h" + + namespace sentencepiece::normalizer { + +diff --git a/src/char_model.cc b/src/char_model.cc +index e06fde3..3cf9860 100644 +--- a/src/char_model.cc ++++ b/src/char_model.cc +@@ -14,7 +14,7 @@ + + #include "char_model.h" + +-#include "third_party/absl/strings/string_view.h" ++#include "absl/strings/string_view.h" + #include "util.h" + + namespace sentencepiece { +diff --git a/src/char_model.h b/src/char_model.h +index 0af17d8..5fe22b5 100644 +--- a/src/char_model.h ++++ b/src/char_model.h +@@ -17,7 +17,7 @@ + + #include "model_interface.h" + #include "sentencepiece_model.pb.h" +-#include "third_party/absl/strings/string_view.h" ++#include "absl/strings/string_view.h" + + namespace sentencepiece { + namespace character { +diff --git a/src/char_model_trainer.cc b/src/char_model_trainer.cc +index c35c059..f11be36 100644 +--- a/src/char_model_trainer.cc ++++ b/src/char_model_trainer.cc +@@ -17,7 +17,7 @@ + #include + + #include "char_model.h" +-#include "third_party/absl/status/status.h" ++#include "absl/status/status.h" + #include "util.h" + + namespace sentencepiece { +diff --git a/src/char_model_trainer.h b/src/char_model_trainer.h +index 01c59fc..119aba8 100644 +--- a/src/char_model_trainer.h ++++ b/src/char_model_trainer.h +@@ -16,7 +16,7 @@ + #define CHAR_MODEL_TRAINER_H_ + + #include "sentencepiece_model.pb.h" +-#include "third_party/absl/status/status.h" ++#include "absl/status/status.h" + #include "trainer_interface.h" + + namespace sentencepiece { +diff --git a/src/common.h b/src/common.h +index ba82860..8a251ab 100644 +--- a/src/common.h ++++ b/src/common.h +@@ -18,12 +18,12 @@ + #include + #include + +-#include "third_party/absl/log/check.h" +-#include "third_party/absl/log/globals.h" +-#include "third_party/absl/log/log.h" +-#include "third_party/absl/status/status.h" +-#include "third_party/absl/status/status_macros.h" +-#include "third_party/absl/strings/string_view.h" ++#include "absl/log/check.h" ++#include "absl/log/globals.h" ++#include "absl/log/log.h" ++#include "absl/status/status.h" ++#include "absl/status/status_macros.h" ++#include "absl/strings/string_view.h" + + #if defined(_WIN32) && !defined(__CYGWIN__) + #define OS_WIN +diff --git a/src/filesystem.cc b/src/filesystem.cc +index 08c2656..174b6bf 100644 +--- a/src/filesystem.cc ++++ b/src/filesystem.cc +@@ -18,8 +18,8 @@ + #include + #include + +-#include "third_party/absl/status/status.h" +-#include "third_party/absl/strings/string_view.h" ++#include "absl/status/status.h" ++#include "absl/strings/string_view.h" + #include "util.h" + + #if defined(OS_WIN) && defined(UNICODE) && defined(_UNICODE) +diff --git a/src/filesystem.h b/src/filesystem.h +index 35f4639..54949c2 100644 +--- a/src/filesystem.h ++++ b/src/filesystem.h +@@ -23,8 +23,8 @@ + + #include "common.h" + #include "sentencepiece_processor.h" +-#include "third_party/absl/status/status.h" +-#include "third_party/absl/strings/string_view.h" ++#include "absl/status/status.h" ++#include "absl/strings/string_view.h" + + namespace sentencepiece { + namespace filesystem { +diff --git a/src/init.cc b/src/init.cc +index c2c651e..641d971 100644 +--- a/src/init.cc ++++ b/src/init.cc +@@ -16,12 +16,12 @@ + + #include "common.h" + #include "config.h" +-#include "third_party/absl/flags/flag.h" +-#include "third_party/absl/flags/parse.h" +-#include "third_party/absl/flags/usage.h" +-#include "third_party/absl/flags/usage_config.h" +-#include "third_party/absl/log/initialize.h" +-#include "third_party/absl/strings/str_cat.h" ++#include "absl/flags/flag.h" ++#include "absl/flags/parse.h" ++#include "absl/flags/usage.h" ++#include "absl/flags/usage_config.h" ++#include "absl/log/initialize.h" ++#include "absl/strings/str_cat.h" + #include "util.h" + + #ifdef _USE_EXTERNAL_PROTOBUF +diff --git a/src/model_interface.cc b/src/model_interface.cc +index 63eccee..f683206 100644 +--- a/src/model_interface.cc ++++ b/src/model_interface.cc +@@ -17,9 +17,9 @@ + #include + + #include "sentencepiece_model.pb.h" +-#include "third_party/absl/strings/str_cat.h" +-#include "third_party/absl/strings/str_format.h" +-#include "third_party/absl/strings/string_view.h" ++#include "absl/strings/str_cat.h" ++#include "absl/strings/str_format.h" ++#include "absl/strings/string_view.h" + #include "util.h" + + namespace sentencepiece { +diff --git a/src/model_interface.h b/src/model_interface.h +index 39e0ef6..25b5dbd 100644 +--- a/src/model_interface.h ++++ b/src/model_interface.h +@@ -25,10 +25,10 @@ + #include "normalizer.h" + #include "sentencepiece_model.pb.h" + #include "sentencepiece_processor.h" +-#include "third_party/absl/container/flat_hash_map.h" +-#include "third_party/absl/status/status.h" +-#include "third_party/absl/strings/string_view.h" +-#include "third_party/darts_clone/darts.h" ++#include "absl/container/flat_hash_map.h" ++#include "absl/status/status.h" ++#include "absl/strings/string_view.h" ++#include "darts.h" + #include "util.h" + + namespace sentencepiece { +diff --git a/src/normalizer.cc b/src/normalizer.cc +index 05a4931..c48778b 100644 +--- a/src/normalizer.cc ++++ b/src/normalizer.cc +@@ -19,11 +19,11 @@ + #include + + #include "common.h" +-#include "third_party/absl/status/status.h" +-#include "third_party/absl/strings/match.h" +-#include "third_party/absl/strings/string_view.h" +-#include "third_party/absl/strings/strip.h" +-#include "third_party/darts_clone/darts.h" ++#include "absl/status/status.h" ++#include "absl/strings/match.h" ++#include "absl/strings/string_view.h" ++#include "absl/strings/strip.h" ++#include "darts.h" + #include "util.h" + + namespace sentencepiece { +diff --git a/src/normalizer.h b/src/normalizer.h +index 67a430b..196bf8d 100644 +--- a/src/normalizer.h ++++ b/src/normalizer.h +@@ -24,9 +24,9 @@ + #include "common.h" + #include "sentencepiece_model.pb.h" + #include "sentencepiece_processor.h" +-#include "third_party/absl/status/status.h" +-#include "third_party/absl/strings/string_view.h" +-#include "third_party/darts_clone/darts.h" ++#include "absl/status/status.h" ++#include "absl/strings/string_view.h" ++#include "darts.h" + + namespace sentencepiece::normalizer { + +diff --git a/src/pretokenizer_for_training.cc b/src/pretokenizer_for_training.cc +index 06debb2..f1da565 100644 +--- a/src/pretokenizer_for_training.cc ++++ b/src/pretokenizer_for_training.cc +@@ -15,8 +15,8 @@ + + #include + +-#include "third_party/absl/strings/str_replace.h" +-#include "third_party/absl/strings/string_view.h" ++#include "absl/strings/str_replace.h" ++#include "absl/strings/string_view.h" + + namespace sentencepiece { + namespace pretokenizer { +diff --git a/src/pretokenizer_for_training.h b/src/pretokenizer_for_training.h +index caa8d3b..20d6924 100644 +--- a/src/pretokenizer_for_training.h ++++ b/src/pretokenizer_for_training.h +@@ -21,8 +21,8 @@ + #include "common.h" + #include "sentencepiece.pb.h" + #include "sentencepiece_processor.h" +-#include "third_party/absl/status/status.h" +-#include "third_party/absl/strings/string_view.h" ++#include "absl/status/status.h" ++#include "absl/strings/string_view.h" + + namespace sentencepiece::pretokenizer { + +diff --git a/src/sentencepiece_processor.cc b/src/sentencepiece_processor.cc +index 1421c18..6c398ee 100644 +--- a/src/sentencepiece_processor.cc ++++ b/src/sentencepiece_processor.cc +@@ -30,20 +30,20 @@ + #include "model_interface.h" + #include "normalizer.h" + #include "sentencepiece.pb.h" +-#include "third_party/absl/cleanup/cleanup.h" +-#include "third_party/absl/container/fixed_array.h" +-#include "third_party/absl/container/flat_hash_set.h" +-#include "third_party/absl/functional/function_ref.h" +-#include "third_party/absl/status/status.h" +-#include "third_party/absl/strings/numbers.h" +-#include "third_party/absl/strings/str_cat.h" +-#include "third_party/absl/strings/str_join.h" +-#include "third_party/absl/strings/str_replace.h" +-#include "third_party/absl/strings/str_split.h" +-#include "third_party/absl/strings/string_view.h" +-#include "third_party/absl/strings/strip.h" +-#include "third_party/absl/synchronization/blocking_counter.h" +-#include "third_party/absl/synchronization/mutex.h" ++#include "absl/cleanup/cleanup.h" ++#include "absl/container/fixed_array.h" ++#include "absl/container/flat_hash_set.h" ++#include "absl/functional/function_ref.h" ++#include "absl/status/status.h" ++#include "absl/strings/numbers.h" ++#include "absl/strings/str_cat.h" ++#include "absl/strings/str_join.h" ++#include "absl/strings/str_replace.h" ++#include "absl/strings/str_split.h" ++#include "absl/strings/string_view.h" ++#include "absl/strings/strip.h" ++#include "absl/synchronization/blocking_counter.h" ++#include "absl/synchronization/mutex.h" + #include "unigram_model.h" + #include "util.h" + +diff --git a/src/sentencepiece_processor.h b/src/sentencepiece_processor.h +index 2e73389..5b67188 100644 +--- a/src/sentencepiece_processor.h ++++ b/src/sentencepiece_processor.h +@@ -24,13 +24,13 @@ + #include + #include + +-#include "third_party/absl/strings/string_view.h" +-#include "third_party/absl/types/span.h" ++#include "absl/strings/string_view.h" ++#include "absl/types/span.h" + namespace absl { + using std::string_view; + } // namespace absl + +-#include "third_party/absl/status/status.h" ++#include "absl/status/status.h" + + namespace sentencepiece { + +diff --git a/src/sentencepiece_trainer.cc b/src/sentencepiece_trainer.cc +index 9898017..65ff242 100644 +--- a/src/sentencepiece_trainer.cc ++++ b/src/sentencepiece_trainer.cc +@@ -23,13 +23,13 @@ + #include "sentencepiece.pb.h" + #include "sentencepiece_model.pb.h" + #include "spec_parser.h" +-#include "third_party/absl/flags/flag.h" +-#include "third_party/absl/status/status.h" +-#include "third_party/absl/strings/numbers.h" +-#include "third_party/absl/strings/str_cat.h" +-#include "third_party/absl/strings/str_split.h" +-#include "third_party/absl/strings/string_view.h" +-#include "third_party/absl/strings/strip.h" ++#include "absl/flags/flag.h" ++#include "absl/status/status.h" ++#include "absl/strings/numbers.h" ++#include "absl/strings/str_cat.h" ++#include "absl/strings/str_split.h" ++#include "absl/strings/string_view.h" ++#include "absl/strings/strip.h" + #include "trainer_factory.h" + #include "util.h" + +diff --git a/src/sentencepiece_trainer.h b/src/sentencepiece_trainer.h +index 5e8d6f7..166c68b 100644 +--- a/src/sentencepiece_trainer.h ++++ b/src/sentencepiece_trainer.h +@@ -21,9 +21,9 @@ + #include + + #include "sentencepiece_processor.h" +-#include "third_party/absl/status/status.h" +-#include "third_party/absl/strings/string_view.h" +-#include "third_party/absl/types/span.h" ++#include "absl/status/status.h" ++#include "absl/strings/string_view.h" ++#include "absl/types/span.h" + + namespace sentencepiece { + +diff --git a/src/spec_parser.h b/src/spec_parser.h +index 41fb573..f5e48b0 100644 +--- a/src/spec_parser.h ++++ b/src/spec_parser.h +@@ -19,12 +19,12 @@ + #include + + #include "sentencepiece_processor.h" +-#include "third_party/absl/status/status.h" +-#include "third_party/absl/status/status_builder.h" +-#include "third_party/absl/strings/ascii.h" +-#include "third_party/absl/strings/numbers.h" +-#include "third_party/absl/strings/str_split.h" +-#include "third_party/absl/strings/string_view.h" ++#include "absl/status/status.h" ++#include "absl/status/status_builder.h" ++#include "absl/strings/ascii.h" ++#include "absl/strings/numbers.h" ++#include "absl/strings/str_split.h" ++#include "absl/strings/string_view.h" + #include "util.h" + + namespace sentencepiece { +diff --git a/src/spm_decode_main.cc b/src/spm_decode_main.cc +index dc5ca5c..97cde99 100644 +--- a/src/spm_decode_main.cc ++++ b/src/spm_decode_main.cc +@@ -21,8 +21,8 @@ + #include "init.h" + #include "sentencepiece.pb.h" + #include "sentencepiece_processor.h" +-#include "third_party/absl/flags/flag.h" +-#include "third_party/absl/strings/str_split.h" ++#include "absl/flags/flag.h" ++#include "absl/strings/str_split.h" + #include "util.h" + + ABSL_FLAG(std::string, model, "", "model file name"); +diff --git a/src/spm_encode_main.cc b/src/spm_encode_main.cc +index b05724a..44328f7 100644 +--- a/src/spm_encode_main.cc ++++ b/src/spm_encode_main.cc +@@ -22,11 +22,11 @@ + #include "init.h" + #include "sentencepiece.pb.h" + #include "sentencepiece_processor.h" +-#include "third_party/absl/container/flat_hash_map.h" +-#include "third_party/absl/flags/flag.h" +-#include "third_party/absl/strings/str_cat.h" +-#include "third_party/absl/strings/str_join.h" +-#include "third_party/absl/strings/string_view.h" ++#include "absl/container/flat_hash_map.h" ++#include "absl/flags/flag.h" ++#include "absl/strings/str_cat.h" ++#include "absl/strings/str_join.h" ++#include "absl/strings/string_view.h" + #include "trainer_interface.h" + + ABSL_FLAG(std::string, model, "", "model file name"); +diff --git a/src/spm_export_vocab_main.cc b/src/spm_export_vocab_main.cc +index f68baa4..ed3c577 100644 +--- a/src/spm_export_vocab_main.cc ++++ b/src/spm_export_vocab_main.cc +@@ -19,7 +19,7 @@ + #include "init.h" + #include "sentencepiece_model.pb.h" + #include "sentencepiece_processor.h" +-#include "third_party/absl/flags/flag.h" ++#include "absl/flags/flag.h" + + ABSL_FLAG(std::string, output, "", "Output filename"); + ABSL_FLAG(std::string, model, "", "input model file name"); +diff --git a/src/spm_normalize_main.cc b/src/spm_normalize_main.cc +index c004189..e15fa32 100644 +--- a/src/spm_normalize_main.cc ++++ b/src/spm_normalize_main.cc +@@ -21,7 +21,7 @@ + #include "sentencepiece_model.pb.h" + #include "sentencepiece_processor.h" + #include "sentencepiece_trainer.h" +-#include "third_party/absl/flags/flag.h" ++#include "absl/flags/flag.h" + + ABSL_FLAG(std::string, model, "", "Model file name"); + ABSL_FLAG(bool, use_internal_normalization, false, +diff --git a/src/spm_train_main.cc b/src/spm_train_main.cc +index f42218e..7218bbc 100644 +--- a/src/spm_train_main.cc ++++ b/src/spm_train_main.cc +@@ -18,11 +18,11 @@ + #include "init.h" + #include "sentencepiece_model.pb.h" + #include "sentencepiece_trainer.h" +-#include "third_party/absl/flags/flag.h" +-#include "third_party/absl/strings/ascii.h" +-#include "third_party/absl/strings/str_join.h" +-#include "third_party/absl/strings/str_split.h" +-#include "third_party/absl/strings/string_view.h" ++#include "absl/flags/flag.h" ++#include "absl/strings/ascii.h" ++#include "absl/strings/str_join.h" ++#include "absl/strings/str_split.h" ++#include "absl/strings/string_view.h" + #include "util.h" + + using sentencepiece::NormalizerSpec; +diff --git a/src/trainer_interface.cc b/src/trainer_interface.cc +index cf89c0a..6570dab 100644 +--- a/src/trainer_interface.cc ++++ b/src/trainer_interface.cc +@@ -29,17 +29,17 @@ + #include "normalizer.h" + #include "sentencepiece_processor.h" + #include "sentencepiece_trainer.h" +-#include "third_party/absl/container/flat_hash_map.h" +-#include "third_party/absl/container/flat_hash_set.h" +-#include "third_party/absl/random/random.h" +-#include "third_party/absl/status/status.h" +-#include "third_party/absl/strings/match.h" +-#include "third_party/absl/strings/numbers.h" +-#include "third_party/absl/strings/str_cat.h" +-#include "third_party/absl/strings/str_format.h" +-#include "third_party/absl/strings/str_join.h" +-#include "third_party/absl/strings/str_split.h" +-#include "third_party/absl/strings/string_view.h" ++#include "absl/container/flat_hash_map.h" ++#include "absl/container/flat_hash_set.h" ++#include "absl/random/random.h" ++#include "absl/status/status.h" ++#include "absl/strings/match.h" ++#include "absl/strings/numbers.h" ++#include "absl/strings/str_cat.h" ++#include "absl/strings/str_format.h" ++#include "absl/strings/str_join.h" ++#include "absl/strings/str_split.h" ++#include "absl/strings/string_view.h" + #include "unicode_script.h" + #include "util.h" + +diff --git a/src/trainer_interface.h b/src/trainer_interface.h +index 9b84a8c..78115aa 100644 +--- a/src/trainer_interface.h ++++ b/src/trainer_interface.h +@@ -27,9 +27,9 @@ + #include "sentencepiece_model.pb.h" + #include "sentencepiece_processor.h" + #include "sentencepiece_trainer.h" +-#include "third_party/absl/container/flat_hash_map.h" +-#include "third_party/absl/status/status.h" +-#include "third_party/absl/strings/string_view.h" ++#include "absl/container/flat_hash_map.h" ++#include "absl/status/status.h" ++#include "absl/strings/string_view.h" + #include "util.h" + + namespace sentencepiece { +diff --git a/src/unicode_script.cc b/src/unicode_script.cc +index c013b4a..5e2b645 100644 +--- a/src/unicode_script.cc ++++ b/src/unicode_script.cc +@@ -14,7 +14,7 @@ + + #include "unicode_script.h" + +-#include "third_party/absl/container/flat_hash_map.h" ++#include "absl/container/flat_hash_map.h" + #include "unicode_script_map.h" + #include "util.h" + +diff --git a/src/unicode_script_map.h b/src/unicode_script_map.h +index a9cf471..2abb706 100644 +--- a/src/unicode_script_map.h ++++ b/src/unicode_script_map.h +@@ -14,7 +14,7 @@ + + #ifndef UNICODE_SCRIPT_DATA_H_ + #define UNICODE_SCRIPT_DATA_H_ +-#include "third_party/absl/container/flat_hash_map.h" ++#include "absl/container/flat_hash_map.h" + namespace sentencepiece { + namespace unicode_script { + namespace { +diff --git a/src/unigram_model.cc b/src/unigram_model.cc +index e6d81c5..2cd4829 100644 +--- a/src/unigram_model.cc ++++ b/src/unigram_model.cc +@@ -24,12 +24,12 @@ + #include + #include + +-#include "third_party/absl/container/flat_hash_map.h" +-#include "third_party/absl/random/random.h" +-#include "third_party/absl/strings/str_split.h" +-#include "third_party/absl/strings/string_view.h" +-#include "third_party/absl/time/clock.h" +-#include "third_party/absl/time/time.h" ++#include "absl/container/flat_hash_map.h" ++#include "absl/random/random.h" ++#include "absl/strings/str_split.h" ++#include "absl/strings/string_view.h" ++#include "absl/time/clock.h" ++#include "absl/time/time.h" + #include "util.h" + + namespace sentencepiece { +diff --git a/src/unigram_model.h b/src/unigram_model.h +index 027b468..5a9c903 100644 +--- a/src/unigram_model.h ++++ b/src/unigram_model.h +@@ -24,8 +24,8 @@ + #include "freelist.h" + #include "model_interface.h" + #include "sentencepiece_model.pb.h" +-#include "third_party/absl/strings/string_view.h" +-#include "third_party/darts_clone/darts.h" ++#include "absl/strings/string_view.h" ++#include "darts.h" + + namespace sentencepiece { + namespace unigram { +diff --git a/src/unigram_model_trainer.cc b/src/unigram_model_trainer.cc +index 13a637e..d2e8b60 100644 +--- a/src/unigram_model_trainer.cc ++++ b/src/unigram_model_trainer.cc +@@ -28,13 +28,13 @@ + #include "normalizer.h" + #include "pretokenizer_for_training.h" + #include "sentencepiece_trainer.h" +-#include "third_party/absl/container/flat_hash_map.h" +-#include "third_party/absl/status/status.h" +-#include "third_party/absl/strings/numbers.h" +-#include "third_party/absl/strings/str_replace.h" +-#include "third_party/absl/strings/str_split.h" +-#include "third_party/absl/strings/string_view.h" +-#include "third_party/esaxx/esa.hxx" // Suffix array library. ++#include "absl/container/flat_hash_map.h" ++#include "absl/status/status.h" ++#include "absl/strings/numbers.h" ++#include "absl/strings/str_replace.h" ++#include "absl/strings/str_split.h" ++#include "absl/strings/string_view.h" ++#include "esa.hxx" // Suffix array library. + #include "trainer_interface.h" + #include "unicode_script.h" + #include "util.h" +diff --git a/src/unigram_model_trainer.h b/src/unigram_model_trainer.h +index 9c3b589..cee64f9 100644 +--- a/src/unigram_model_trainer.h ++++ b/src/unigram_model_trainer.h +@@ -21,8 +21,8 @@ + #include + + #include "sentencepiece_model.pb.h" +-#include "third_party/absl/status/status.h" +-#include "third_party/absl/strings/string_view.h" ++#include "absl/status/status.h" ++#include "absl/strings/string_view.h" + #include "trainer_interface.h" + #include "unigram_model.h" + #include "util.h" +diff --git a/src/util.cc b/src/util.cc +index bd5ea1a..d5f5532 100644 +--- a/src/util.cc ++++ b/src/util.cc +@@ -18,11 +18,11 @@ + #include + #include + +-#include "third_party/absl/random/random.h" +-#include "third_party/absl/status/status.h" +-#include "third_party/absl/strings/string_view.h" +-#include "third_party/absl/synchronization/blocking_counter.h" +-#include "third_party/absl/synchronization/mutex.h" ++#include "absl/random/random.h" ++#include "absl/status/status.h" ++#include "absl/strings/string_view.h" ++#include "absl/synchronization/blocking_counter.h" ++#include "absl/synchronization/mutex.h" + + namespace sentencepiece { + +diff --git a/src/util.h b/src/util.h +index 07d9f37..63b4873 100644 +--- a/src/util.h ++++ b/src/util.h +@@ -32,21 +32,21 @@ + #include "common.h" + #include "config.h" + #include "sentencepiece_processor.h" +-#include "third_party/absl/base/internal/endian.h" +-#include "third_party/absl/base/thread_annotations.h" +-#include "third_party/absl/functional/any_invocable.h" +-#include "third_party/absl/numeric/bits.h" +-#include "third_party/absl/random/random.h" +-#include "third_party/absl/status/status.h" +-#include "third_party/absl/status/status_builder.h" +-#include "third_party/absl/strings/ascii.h" +-#include "third_party/absl/strings/numbers.h" +-#include "third_party/absl/strings/str_cat.h" +-#include "third_party/absl/strings/str_format.h" +-#include "third_party/absl/strings/str_join.h" +-#include "third_party/absl/strings/string_view.h" +-#include "third_party/absl/strings/strip.h" +-#include "third_party/absl/synchronization/mutex.h" ++#include "absl/base/internal/endian.h" ++#include "absl/base/thread_annotations.h" ++#include "absl/functional/any_invocable.h" ++#include "absl/numeric/bits.h" ++#include "absl/random/random.h" ++#include "absl/status/status.h" ++#include "absl/status/status_builder.h" ++#include "absl/strings/ascii.h" ++#include "absl/strings/numbers.h" ++#include "absl/strings/str_cat.h" ++#include "absl/strings/str_format.h" ++#include "absl/strings/str_join.h" ++#include "absl/strings/string_view.h" ++#include "absl/strings/strip.h" ++#include "absl/synchronization/mutex.h" + + static constexpr uint32_t kUnicodeError = 0xFFFD; + +diff --git a/src/word_model.cc b/src/word_model.cc +index aa69f40..7868931 100644 +--- a/src/word_model.cc ++++ b/src/word_model.cc +@@ -14,7 +14,7 @@ + + #include "word_model.h" + +-#include "third_party/absl/strings/string_view.h" ++#include "absl/strings/string_view.h" + #include "util.h" + + namespace sentencepiece { +diff --git a/src/word_model.h b/src/word_model.h +index be1d034..7ea554c 100644 +--- a/src/word_model.h ++++ b/src/word_model.h +@@ -17,7 +17,7 @@ + + #include "model_interface.h" + #include "sentencepiece_model.pb.h" +-#include "third_party/absl/strings/string_view.h" ++#include "absl/strings/string_view.h" + + namespace sentencepiece { + namespace word { +diff --git a/src/word_model_trainer.cc b/src/word_model_trainer.cc +index 4bea35e..005a162 100644 +--- a/src/word_model_trainer.cc ++++ b/src/word_model_trainer.cc +@@ -17,9 +17,9 @@ + #include + #include + +-#include "third_party/absl/container/flat_hash_map.h" +-#include "third_party/absl/status/status.h" +-#include "third_party/absl/strings/string_view.h" ++#include "absl/container/flat_hash_map.h" ++#include "absl/status/status.h" ++#include "absl/strings/string_view.h" + #include "util.h" + #include "word_model.h" + +diff --git a/src/word_model_trainer.h b/src/word_model_trainer.h +index 77f2457..19ff9f5 100644 +--- a/src/word_model_trainer.h ++++ b/src/word_model_trainer.h +@@ -16,7 +16,7 @@ + #define WORD_MODEL_TRAINER_H_ + + #include "sentencepiece_model.pb.h" +-#include "third_party/absl/status/status.h" ++#include "absl/status/status.h" + #include "trainer_interface.h" + + namespace sentencepiece { diff --git a/modules/sentencepiece/0.2.2/presubmit.yml b/modules/sentencepiece/0.2.2/presubmit.yml new file mode 100644 index 00000000000..f2d095f9de1 --- /dev/null +++ b/modules/sentencepiece/0.2.2/presubmit.yml @@ -0,0 +1,92 @@ +matrix: + unix_platform: + - debian11 + - ubuntu2404 + - ubuntu2404_arm64 + - macos + - macos_arm64 + windows_platform: + - windows + bazel: + - 8.x + - 9.x +tasks: + verify_targets_unix: + name: Verify build targets (Unix) + platform: ${{ unix_platform }} + bazel: ${{ bazel }} + environment: + ANDROID_HOME: "" + ANDROID_NDK_HOME: "" + build_flags: + - '--cxxopt=-std=c++17' + - '--host_cxxopt=-std=c++17' + build_targets: + - '@sentencepiece//:sentencepiece' + - '@sentencepiece//:sentencepiece_train' + - '@sentencepiece//src:spm_decode' + - '@sentencepiece//src:spm_encode' + - '@sentencepiece//src:spm_export_vocab' + - '@sentencepiece//src:spm_normalize' + - '@sentencepiece//src:spm_train' + verify_targets_windows: + name: Verify build targets (Windows) + platform: ${{ windows_platform }} + bazel: ${{ bazel }} + environment: + ANDROID_HOME: "" + ANDROID_NDK_HOME: "" + build_flags: + - '--cxxopt=/std:c++17' + - '--host_cxxopt=/std:c++17' + - '--cxxopt=/utf-8' + - '--host_cxxopt=/utf-8' + build_targets: + - '@sentencepiece//:sentencepiece' + - '@sentencepiece//:sentencepiece_train' + - '@sentencepiece//src:spm_decode' + - '@sentencepiece//src:spm_encode' + - '@sentencepiece//src:spm_export_vocab' + - '@sentencepiece//src:spm_normalize' + - '@sentencepiece//src:spm_train' +bcr_test_module: + module_path: bcr_tests + matrix: + unix_platform: + - debian11 + - ubuntu2404 + - ubuntu2404_arm64 + - macos + - macos_arm64 + windows_platform: + - windows + bazel: + - 8.x + - 9.x + tasks: + run_test_module_unix: + name: Run test module (Unix) + platform: ${{ unix_platform }} + bazel: ${{ bazel }} + environment: + ANDROID_HOME: "" + ANDROID_NDK_HOME: "" + build_flags: + - '--cxxopt=-std=c++17' + - '--host_cxxopt=-std=c++17' + test_targets: + - '//...' + run_test_module_windows: + name: Run test module (Windows) + platform: ${{ windows_platform }} + bazel: ${{ bazel }} + environment: + ANDROID_HOME: "" + ANDROID_NDK_HOME: "" + build_flags: + - '--cxxopt=/std:c++17' + - '--host_cxxopt=/std:c++17' + - '--cxxopt=/utf-8' + - '--host_cxxopt=/utf-8' + test_targets: + - '//...' diff --git a/modules/sentencepiece/0.2.2/source.json b/modules/sentencepiece/0.2.2/source.json new file mode 100644 index 00000000000..ce2c6fb2c47 --- /dev/null +++ b/modules/sentencepiece/0.2.2/source.json @@ -0,0 +1,16 @@ +{ + "url": "https://github.com/google/sentencepiece/releases/download/v0.2.2/sentencepiece-0.2.2.tar.gz", + "integrity": "sha256-PStegktWIgONx7SQiX7+Beu7uec1D8FC8+zIeJ75vfY=", + "strip_prefix": "sentencepiece-0.2.2/sentencepiece", + "patch_strip": 1, + "overlay": { + "BUILD.bazel": "sha256-1J6GYEpKhJYzSsidqRT3qJ8s8DWbkt5Da2teqqKokgU=", + "bcr_tests/BUILD.bazel": "sha256-wAPfmKxEAhDErWGI/2r+GlMO6Y8kh4jGgmO82j9mR9M=", + "bcr_tests/MODULE.bazel": "sha256-korE24ctlBVWC6o2iBNFv4eEConbhnQ7t24f/pONHrE=", + "bcr_tests/roundtrip_test.cc": "sha256-P+ODDbBogNbZKvbPuG/s3gGi5y3oWZcLgYHxxcH7pm0=", + "src/BUILD.bazel": "sha256-y+FvhIuVeC9xNfn1T69V57N9jsvvVlzC3Wm1kmgrkaw=" + }, + "patches": { + "use_bcr_dependency_includes.patch": "sha256-Bzf37JsWUvZ/CYG/X1fey6ZJU8HLvJGsUxcX2hgIpmw=" + } +} diff --git a/modules/sentencepiece/metadata.json b/modules/sentencepiece/metadata.json index f9e04415412..354011bb752 100644 --- a/modules/sentencepiece/metadata.json +++ b/modules/sentencepiece/metadata.json @@ -12,7 +12,8 @@ "github:google/sentencepiece" ], "versions": [ - "0.2.1" + "0.2.1", + "0.2.2" ], "yanked_versions": {} }