From 6f35a0a31a254716476e530fc6eda316ec1f667f Mon Sep 17 00:00:00 2001 From: mohmedmm <48658166+mohmedmm@users.noreply.github.com> Date: Sat, 5 Sep 2026 17:40:54 +0200 Subject: [PATCH 1/5] Fix padding configuration in preprocessor_utils.py --- fastembed/common/preprocessor_utils.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/fastembed/common/preprocessor_utils.py b/fastembed/common/preprocessor_utils.py index 3b702f799..e899700e9 100644 --- a/fastembed/common/preprocessor_utils.py +++ b/fastembed/common/preprocessor_utils.py @@ -51,8 +51,19 @@ def load_tokenizer(model_dir: Path) -> tuple[Tokenizer, dict[str, int]]: tokenizer = Tokenizer.from_file(str(tokenizer_path)) tokenizer.enable_truncation(max_length=max_context) if not tokenizer.padding: + pad_to_multiple_of = ( + tokenizer_config.get("pad_to_multiple_of") + if tokenizer_config.get("pad_to_multiple_of") is not None + else ( + config.get("pad_to_multiple_of") + if config.get("pad_to_multiple_of") is not None + else tokenizer_config.get("pad_to_multiple_of") + ) + ) tokenizer.enable_padding( - pad_id=config.get("pad_token_id", 0), pad_token=tokenizer_config["pad_token"] + pad_id=config.get("pad_token_id", 0), + pad_token=tokenizer_config["pad_token"], + pad_to_multiple_of=pad_to_multiple_of, ) for token in tokens_map.values(): From e6db57a87d54266907e3ace9dc367561e8f4c6c5 Mon Sep 17 00:00:00 2001 From: mohmedmm <48658166+mohmedmm@users.noreply.github.com> Date: Sat, 5 Sep 2026 17:42:09 +0200 Subject: [PATCH 2/5] Add test for loading tokenizer with pad_to_multiple_of Added a test to verify loading of tokenizer with padding configuration. --- tests/test_common.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tests/test_common.py b/tests/test_common.py index f7cae5ba5..0b8bd98d4 100644 --- a/tests/test_common.py +++ b/tests/test_common.py @@ -59,3 +59,28 @@ def test_last_token_pooling_with_left_padding(): pooled = last_token_pooling(token_embeddings, attention_mask) assert np.allclose(pooled, [[2.0, 2.0], [6.0, 6.0]]) + + +def test_load_tokenizer_with_pad_to_multiple_of(tmp_path): + import json + from fastembed.common.preprocessor_utils import load_tokenizer + + model_dir = tmp_path / "model" + model_dir.mkdir() + + (model_dir / "config.json").write_text(json.dumps({"pad_token_id": 0})) + (model_dir / "tokenizer_config.json").write_text( + json.dumps({"model_max_length": 128, "pad_token": "", "pad_to_multiple_of": 8}) + ) + (model_dir / "special_tokens_map.json").write_text(json.dumps({})) + + from tokenizers import Tokenizer + from tokenizers.models import BPE + + tokenizer = Tokenizer(BPE()) + tokenizer.save(str(model_dir / "tokenizer.json")) + + loaded_tokenizer, _ = load_tokenizer(model_dir) + + assert loaded_tokenizer.padding is not None + assert loaded_tokenizer.padding.get("pad_to_multiple_of") == 8 From 498681bb853a2029f0f67ba790c475f41d85fcbc Mon Sep 17 00:00:00 2001 From: mohmedmm <48658166+mohmedmm@users.noreply.github.com> Date: Sat, 5 Sep 2026 17:45:39 +0200 Subject: [PATCH 3/5] Improve tokenizer padding handling Refactor padding logic in tokenizer configuration. --- fastembed/common/preprocessor_utils.py | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/fastembed/common/preprocessor_utils.py b/fastembed/common/preprocessor_utils.py index e899700e9..2a414c639 100644 --- a/fastembed/common/preprocessor_utils.py +++ b/fastembed/common/preprocessor_utils.py @@ -1,6 +1,6 @@ import json -from typing import Any from pathlib import Path +from typing import Any from tokenizers import AddedToken, Tokenizer @@ -51,19 +51,18 @@ def load_tokenizer(model_dir: Path) -> tuple[Tokenizer, dict[str, int]]: tokenizer = Tokenizer.from_file(str(tokenizer_path)) tokenizer.enable_truncation(max_length=max_context) if not tokenizer.padding: - pad_to_multiple_of = ( - tokenizer_config.get("pad_to_multiple_of") - if tokenizer_config.get("pad_to_multiple_of") is not None - else ( - config.get("pad_to_multiple_of") - if config.get("pad_to_multiple_of") is not None - else tokenizer_config.get("pad_to_multiple_of") - ) + tokenizer.enable_padding( + pad_id=config.get("pad_token_id", 0), pad_token=tokenizer_config["pad_token"] ) + elif tokenizer.padding.get("length") is not None: + padding_params = tokenizer.padding tokenizer.enable_padding( - pad_id=config.get("pad_token_id", 0), - pad_token=tokenizer_config["pad_token"], - pad_to_multiple_of=pad_to_multiple_of, + direction=padding_params.get("direction", "right"), + pad_id=padding_params.get("pad_id", config.get("pad_token_id", 0)), + pad_type_id=padding_params.get("pad_type_id", 0), + pad_token=padding_params.get("pad_token", tokenizer_config.get("pad_token", "[PAD]")), + pad_to_multiple_of=padding_params.get("pad_to_multiple_of"), + length=None, ) for token in tokens_map.values(): From 6fb5e843a0ac6b137bb664ce38feb5fca8c6f468 Mon Sep 17 00:00:00 2001 From: mohmedmm <48658166+mohmedmm@users.noreply.github.com> Date: Sat, 5 Sep 2026 17:46:08 +0200 Subject: [PATCH 4/5] Improve tokenizer test cases and assertions Refactor tokenizer tests for clarity and correctness. --- tests/test_common.py | 93 +++++++++++++++++++++++++++++++++----------- 1 file changed, 71 insertions(+), 22 deletions(-) diff --git a/tests/test_common.py b/tests/test_common.py index 0b8bd98d4..828e4af83 100644 --- a/tests/test_common.py +++ b/tests/test_common.py @@ -1,11 +1,11 @@ import numpy as np from fastembed import ( - TextEmbedding, - SparseTextEmbedding, ImageEmbedding, LateInteractionMultimodalEmbedding, LateInteractionTextEmbedding, + SparseTextEmbedding, + TextEmbedding, ) from fastembed.common.utils import last_token_pooling @@ -23,13 +23,13 @@ def test_text_list_supported_models(): description = supported_models[0] assert isinstance(description, dict) - assert "model" in description and description["model"] + assert description.get("model") if model_type != SparseTextEmbedding: - assert "dim" in description and description["dim"] - assert "license" in description and description["license"] - assert "size_in_GB" in description and description["size_in_GB"] - assert "model_file" in description and description["model_file"] - assert "sources" in description and description["sources"] + assert description.get("dim") + assert description.get("license") + assert description.get("size_in_GB") + assert description.get("model_file") + assert description.get("sources") assert "hf" in description["sources"] or "url" in description["sources"] @@ -61,26 +61,75 @@ def test_last_token_pooling_with_left_padding(): assert np.allclose(pooled, [[2.0, 2.0], [6.0, 6.0]]) -def test_load_tokenizer_with_pad_to_multiple_of(tmp_path): +def test_load_tokenizer_fixed_length_padding_converted_to_dynamic(tmp_path): import json + + from tokenizers import Tokenizer, models + from fastembed.common.preprocessor_utils import load_tokenizer - model_dir = tmp_path / "model" - model_dir.mkdir() + config = {"pad_token_id": 0} + with open(tmp_path / "config.json", "w") as f: + json.dump(config, f) - (model_dir / "config.json").write_text(json.dumps({"pad_token_id": 0})) - (model_dir / "tokenizer_config.json").write_text( - json.dumps({"model_max_length": 128, "pad_token": "", "pad_to_multiple_of": 8}) - ) - (model_dir / "special_tokens_map.json").write_text(json.dumps({})) + tokenizer_config = { + "model_max_length": 512, + "pad_token": "[PAD]", + } + with open(tmp_path / "tokenizer_config.json", "w") as f: + json.dump(tokenizer_config, f) + + with open(tmp_path / "special_tokens_map.json", "w") as f: + json.dump({"pad_token": "[PAD]"}, f) + + # Tokenizer initialized with fixed-length padding (e.g. gte-base with length=128) + tokenizer = Tokenizer(models.BPE()) + tokenizer.add_special_tokens(["[PAD]"]) + tokenizer.enable_padding(length=128, pad_id=0, pad_token="[PAD]", direction="right") + tokenizer.save(str(tmp_path / "tokenizer.json")) + + loaded_tokenizer, _ = load_tokenizer(tmp_path) + + # Fixed length must be relaxed to None (dynamic batch padding) to prevent ragged arrays + assert loaded_tokenizer.padding is not None + assert loaded_tokenizer.padding["length"] is None + assert loaded_tokenizer.padding["direction"] == "right" + assert loaded_tokenizer.padding["pad_id"] == 0 + assert loaded_tokenizer.padding["pad_token"] == "[PAD]" + assert loaded_tokenizer.truncation["max_length"] == 512 + + +def test_load_tokenizer_preserves_left_padding(tmp_path): + import json + + from tokenizers import Tokenizer, models + + from fastembed.common.preprocessor_utils import load_tokenizer + + config = {"pad_token_id": 50283} + with open(tmp_path / "config.json", "w") as f: + json.dump(config, f) + + tokenizer_config = { + "model_max_length": 8192, + "pad_token": "[PAD]", + } + with open(tmp_path / "tokenizer_config.json", "w") as f: + json.dump(tokenizer_config, f) - from tokenizers import Tokenizer - from tokenizers.models import BPE + with open(tmp_path / "special_tokens_map.json", "w") as f: + json.dump({"pad_token": "[PAD]"}, f) - tokenizer = Tokenizer(BPE()) - tokenizer.save(str(model_dir / "tokenizer.json")) + # Tokenizer with dynamic left padding (e.g. ColModernVBERT) + tokenizer = Tokenizer(models.BPE()) + tokenizer.add_special_tokens(["[PAD]"]) + tokenizer.enable_padding(length=None, pad_id=50283, pad_token="[PAD]", direction="left") + tokenizer.save(str(tmp_path / "tokenizer.json")) - loaded_tokenizer, _ = load_tokenizer(model_dir) + loaded_tokenizer, _ = load_tokenizer(tmp_path) + # Preserves left padding direction and pad_id assert loaded_tokenizer.padding is not None - assert loaded_tokenizer.padding.get("pad_to_multiple_of") == 8 + assert loaded_tokenizer.padding["length"] is None + assert loaded_tokenizer.padding["direction"] == "left" + assert loaded_tokenizer.padding["pad_id"] == 50283 From 6bb4944d840abab6effce9880821c5f7a5012ec3 Mon Sep 17 00:00:00 2001 From: Mohammed Alshek Date: Sun, 6 Sep 2026 01:03:46 +0200 Subject: [PATCH 5/5] fix: address code review feedback on padding handling and coverage - Add docstrings to satisfy documentation coverage threshold - Validate pad_to_multiple_of as a positive integer - Support config.json fallback for pad_to_multiple_of - Apply configured pad_to_multiple_of to serialized padding settings - Add comprehensive unit tests covering validation and precedence --- fastembed/common/preprocessor_utils.py | 69 +++++++++++++++++++++---- tests/test_common.py | 71 ++++++++++++++++++++++++++ 2 files changed, 131 insertions(+), 9 deletions(-) diff --git a/fastembed/common/preprocessor_utils.py b/fastembed/common/preprocessor_utils.py index 2a414c639..dedb7e0ad 100644 --- a/fastembed/common/preprocessor_utils.py +++ b/fastembed/common/preprocessor_utils.py @@ -19,6 +19,25 @@ def load_special_tokens(model_dir: Path) -> dict[str, Any]: def load_tokenizer(model_dir: Path) -> tuple[Tokenizer, dict[str, int]]: + """ + Load and configure a tokenizer from a model directory. + + Configures truncation to the model context length, converts any fixed-length + padding to dynamic batch padding (avoiding ragged batch failures), preserves + serialized padding direction and token IDs, and optionally applies padding + multiples (e.g. pad_to_multiple_of) when configured. + + Args: + model_dir: Directory path containing tokenizer configuration files + (config.json, tokenizer.json, tokenizer_config.json, special_tokens_map.json). + + Returns: + A tuple of (configured Tokenizer instance, mapping of special token strings to token IDs). + + Raises: + ValueError: If required configuration files are missing or if pad_to_multiple_of + is not a positive integer. + """ config_path = model_dir / "config.json" if not config_path.exists(): raise ValueError(f"Could not find config.json in {model_dir}") @@ -50,20 +69,52 @@ def load_tokenizer(model_dir: Path) -> tuple[Tokenizer, dict[str, int]]: tokenizer = Tokenizer.from_file(str(tokenizer_path)) tokenizer.enable_truncation(max_length=max_context) + + pad_to_multiple_of = tokenizer_config.get("pad_to_multiple_of") + if pad_to_multiple_of is None: + pad_to_multiple_of = config.get("pad_to_multiple_of") + + if pad_to_multiple_of is not None and ( + not isinstance(pad_to_multiple_of, int) + or isinstance(pad_to_multiple_of, bool) + or pad_to_multiple_of <= 0 + ): + raise ValueError("pad_to_multiple_of must be a positive integer") + if not tokenizer.padding: tokenizer.enable_padding( - pad_id=config.get("pad_token_id", 0), pad_token=tokenizer_config["pad_token"] + pad_id=config.get("pad_token_id", 0), + pad_token=tokenizer_config.get("pad_token", "[PAD]"), + pad_to_multiple_of=pad_to_multiple_of, ) - elif tokenizer.padding.get("length") is not None: + else: padding_params = tokenizer.padding - tokenizer.enable_padding( - direction=padding_params.get("direction", "right"), - pad_id=padding_params.get("pad_id", config.get("pad_token_id", 0)), - pad_type_id=padding_params.get("pad_type_id", 0), - pad_token=padding_params.get("pad_token", tokenizer_config.get("pad_token", "[PAD]")), - pad_to_multiple_of=padding_params.get("pad_to_multiple_of"), - length=None, + target_pad_to_multiple_of = ( + pad_to_multiple_of + if pad_to_multiple_of is not None + else padding_params.get("pad_to_multiple_of") ) + if target_pad_to_multiple_of is not None and ( + not isinstance(target_pad_to_multiple_of, int) + or isinstance(target_pad_to_multiple_of, bool) + or target_pad_to_multiple_of <= 0 + ): + raise ValueError("pad_to_multiple_of must be a positive integer") + + if padding_params.get("length") is not None or ( + pad_to_multiple_of is not None + and padding_params.get("pad_to_multiple_of") != target_pad_to_multiple_of + ): + tokenizer.enable_padding( + direction=padding_params.get("direction", "right"), + pad_id=padding_params.get("pad_id", config.get("pad_token_id", 0)), + pad_type_id=padding_params.get("pad_type_id", 0), + pad_token=padding_params.get( + "pad_token", tokenizer_config.get("pad_token", "[PAD]") + ), + pad_to_multiple_of=target_pad_to_multiple_of, + length=None, + ) for token in tokens_map.values(): if isinstance(token, str): diff --git a/tests/test_common.py b/tests/test_common.py index 828e4af83..744219fa0 100644 --- a/tests/test_common.py +++ b/tests/test_common.py @@ -62,6 +62,10 @@ def test_last_token_pooling_with_left_padding(): def test_load_tokenizer_fixed_length_padding_converted_to_dynamic(tmp_path): + """ + Verify that models with serialized fixed-length padding (e.g. gte-base with length=128) + have their padding relaxed to dynamic batch padding (length=None) to support mixed-length batches. + """ import json from tokenizers import Tokenizer, models @@ -100,6 +104,10 @@ def test_load_tokenizer_fixed_length_padding_converted_to_dynamic(tmp_path): def test_load_tokenizer_preserves_left_padding(tmp_path): + """ + Verify that tokenizers with serialized left padding (e.g. ColModernVBERT) + preserve their padding direction and pad_token_id without being overridden. + """ import json from tokenizers import Tokenizer, models @@ -133,3 +141,66 @@ def test_load_tokenizer_preserves_left_padding(tmp_path): assert loaded_tokenizer.padding["length"] is None assert loaded_tokenizer.padding["direction"] == "left" assert loaded_tokenizer.padding["pad_id"] == 50283 + + +def test_load_tokenizer_pad_to_multiple_of_and_validation(tmp_path): + """ + Verify pad_to_multiple_of configuration precedence (tokenizer_config > config fallback), + application to unpadded and serialized tokenizers, and validation of positive integer values. + """ + import json + + import pytest + from tokenizers import Tokenizer, models + + from fastembed.common.preprocessor_utils import load_tokenizer + + # 1. Test pad_to_multiple_of via config.json fallback + dir_fallback = tmp_path / "fallback" + dir_fallback.mkdir() + (dir_fallback / "config.json").write_text( + json.dumps({"pad_token_id": 0, "pad_to_multiple_of": 16}) + ) + (dir_fallback / "tokenizer_config.json").write_text( + json.dumps({"model_max_length": 128, "pad_token": "[PAD]"}) + ) + (dir_fallback / "special_tokens_map.json").write_text(json.dumps({"pad_token": "[PAD]"})) + tok = Tokenizer(models.BPE()) + tok.add_special_tokens(["[PAD]"]) + tok.save(str(dir_fallback / "tokenizer.json")) + + loaded_tok, _ = load_tokenizer(dir_fallback) + assert loaded_tok.padding is not None + assert loaded_tok.padding.get("pad_to_multiple_of") == 16 + + # 2. Test tokenizer_config.json precedence over config.json + dir_prec = tmp_path / "prec" + dir_prec.mkdir() + (dir_prec / "config.json").write_text( + json.dumps({"pad_token_id": 0, "pad_to_multiple_of": 16}) + ) + (dir_prec / "tokenizer_config.json").write_text( + json.dumps({"model_max_length": 128, "pad_token": "[PAD]", "pad_to_multiple_of": 8}) + ) + (dir_prec / "special_tokens_map.json").write_text(json.dumps({"pad_token": "[PAD]"})) + tok.save(str(dir_prec / "tokenizer.json")) + + loaded_tok, _ = load_tokenizer(dir_prec) + assert loaded_tok.padding is not None + assert loaded_tok.padding.get("pad_to_multiple_of") == 8 + + # 3. Test validation error on invalid pad_to_multiple_of (<= 0 or non-integer) + for invalid_val in [0, -1, "8", False]: + dir_invalid = tmp_path / f"invalid_{invalid_val}" + dir_invalid.mkdir() + (dir_invalid / "config.json").write_text(json.dumps({"pad_token_id": 0})) + (dir_invalid / "tokenizer_config.json").write_text( + json.dumps( + {"model_max_length": 128, "pad_token": "[PAD]", "pad_to_multiple_of": invalid_val} + ) + ) + (dir_invalid / "special_tokens_map.json").write_text(json.dumps({"pad_token": "[PAD]"})) + tok.save(str(dir_invalid / "tokenizer.json")) + + with pytest.raises(ValueError, match="pad_to_multiple_of must be a positive integer"): + load_tokenizer(dir_invalid)