Add sentencepiece@0.2.2

modules/sentencepiece/0.2.2/MODULE.bazel

@@ -0,0 +1,26 @@
+"""SentencePiece is an unsupervised text tokenizer and detokenizer mainly for Neural Network-based text generation systems."""
+
+module(
+    name = "sentencepiece",
+    version = "0.2.2",
+    bazel_compatibility = [">=7.2.1"],
+)
+
+# Keep in sync with the abseil-cpp GIT_TAG in CMakeLists.txt.
+bazel_dep(name = "abseil-cpp", version = "20260526.0")
+
+# Upstream darts-clone v0.32h plus the compatibility helpers (copy_array,
+# validate, ...) that the vendored third_party/darts_clone copy carries.
+bazel_dep(name = "darts-clone", version = "0.32h.bcr.1")
+
+# Upstream esaxx at the commit the vendored third_party/esaxx copy is based
+# on. The .bcr.1 revision exports esa.hxx and carries the index_type(0)
+# template-deduction fix that the int64_t instantiation of esaxx() needs.
+bazel_dep(name = "esaxx", version = "20250106.1.bcr.1")
+bazel_dep(name = "platforms", version = "1.1.0")
+
+# The .pb.cc/.pb.h files are regenerated from src/*.proto at build time, so
+# the Bazel build follows the SPM_PROTOBUF_PROVIDER=package CMake
+# configuration instead of the vendored third_party/protobuf-lite runtime.
+bazel_dep(name = "protobuf", version = "35.1")
+bazel_dep(name = "rules_cc", version = "0.2.18")

modules/sentencepiece/0.2.2/overlay/BUILD.bazel

@@ -0,0 +1,31 @@
+# Copyright 2018 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.!
+
+package(default_visibility = ["//visibility:public"])
+
+exports_files([
+    "VERSION.txt",
+    "config.h.in",
+    "LICENSE",
+])
+
+alias(
+    name = "sentencepiece",
+    actual = "//src:sentencepiece",
+)
+
+alias(
+    name = "sentencepiece_train",
+    actual = "//src:sentencepiece_train",
+)

modules/sentencepiece/0.2.2/overlay/bcr_tests/BUILD.bazel

@@ -0,0 +1,11 @@
+load("@rules_cc//cc:defs.bzl", "cc_test")
+
+cc_test(
+    name = "roundtrip_test",
+    srcs = ["roundtrip_test.cc"],
+    deps = [
+        "@abseil-cpp//absl/status",
+        "@sentencepiece//:sentencepiece",
+        "@sentencepiece//:sentencepiece_train",
+    ],
+)

modules/sentencepiece/0.2.2/overlay/bcr_tests/MODULE.bazel

@@ -0,0 +1,10 @@
+module(name = "sentencepiece_bcr_tests")
+
+bazel_dep(name = "sentencepiece")
+local_path_override(
+    module_name = "sentencepiece",
+    path = "..",
+)
+
+bazel_dep(name = "abseil-cpp", version = "20260526.0")
+bazel_dep(name = "rules_cc", version = "0.2.18")

modules/sentencepiece/0.2.2/overlay/bcr_tests/roundtrip_test.cc

@@ -0,0 +1,66 @@
+// Trains a small model fully in memory, then checks that encoding and
+// decoding roundtrips. This replaces the upstream //src:spm_test, whose
+// test data (data/) is not shipped in the release tarball.
+
+#include <iostream>
+#include <string>
+#include <vector>
+
+#include "absl/status/status.h"
+#include "sentencepiece_processor.h"
+#include "sentencepiece_trainer.h"
+
+#define CHECK_TEST(condition)                                         \
+  if (!(condition)) {                                                 \
+    std::cerr << __FILE__ << ":" << __LINE__ << " check failed: "     \
+              << #condition << std::endl;                             \
+    return 1;                                                         \
+  }
+
+#define CHECK_OK_TEST(expr)                                           \
+  {                                                                   \
+    const absl::Status _status = (expr);                              \
+    if (!_status.ok()) {                                              \
+      std::cerr << __FILE__ << ":" << __LINE__ << " status not ok: "  \
+                << _status << std::endl;                              \
+      return 1;                                                       \
+    }                                                                 \
+  }
+
+int main() {
+  const std::vector<std::string> base = {
+      "The quick brown fox jumps over the lazy dog.",
+      "SentencePiece is an unsupervised text tokenizer and detokenizer.",
+      "Hello world. Hello Bazel.",
+      "I saw a girl with a telescope.",
+  };
+  std::vector<std::string> sentences;
+  for (int i = 0; i < 200; ++i) {
+    sentences.push_back(base[i % base.size()]);
+  }
+
+  std::string serialized_model;
+  CHECK_OK_TEST(sentencepiece::SentencePieceTrainer::Train(
+      "--vocab_size=60 --hard_vocab_limit=false --minloglevel=1", sentences,
+      &serialized_model));
+  CHECK_TEST(!serialized_model.empty());
+
+  sentencepiece::SentencePieceProcessor sp;
+  CHECK_OK_TEST(sp.LoadFromSerializedProto(serialized_model));
+
+  const std::string input = "Hello world.";
+  std::vector<std::string> pieces;
+  CHECK_OK_TEST(sp.Encode(input, &pieces));
+  CHECK_TEST(!pieces.empty());
+
+  std::string detokenized;
+  CHECK_OK_TEST(sp.Decode(pieces, &detokenized));
+  CHECK_TEST(detokenized == input);
+
+  std::vector<int> ids;
+  CHECK_OK_TEST(sp.Encode(input, &ids));
+  CHECK_TEST(!ids.empty());
+
+  std::cout << "PASS" << std::endl;
+  return 0;
+}