Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
55 changes: 51 additions & 4 deletions haystack/document_stores/in_memory/document_store.py
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,44 @@
r"\uff66-\uff9f" # Halfwidth Katakana
r"\uffa0-\uffdc" # Halfwidth Hangul Jamo
)
_DEFAULT_BM25_TOKENIZATION_REGEX = rf"[^\W{_CJK_CHAR_CLASS}]+|[{_CJK_CHAR_CLASS}]"


def _combining_mark_char_class() -> str:
r"""
Build a regex character class body that matches every combining mark (Unicode categories Mn, Mc and Me).

Python's `re` does not count combining marks as word characters, so `\w` alone cuts words apart in every
script that writes vowels and other signs as marks. For example, Sinhala "ජනාධිපති" becomes "ජන", "ධ", "පත",
and Tamil, Devanagari, Bengali and fully vowelled Arabic or Hebrew break the same way. The marks of all modern
scripts are in the Basic Multilingual Plane, so only that plane is scanned. The kana voicing marks
(U+3099-U+309A) are left out so kana keeps its one-token-per-character behaviour.
"""
ranges = []
start = None
for code_point in range(0x10001):
is_mark = (
code_point < 0x10000
and unicodedata.category(chr(code_point)).startswith("M")
and not 0x3099 <= code_point <= 0x309A
)
if is_mark and start is None:
start = code_point
elif not is_mark and start is not None:
end = code_point - 1
ranges.append(rf"\u{start:04x}" if start == end else rf"\u{start:04x}-\u{end:04x}")
start = None
return "".join(ranges)


_COMBINING_MARK_CHAR_CLASS = _combining_mark_char_class()
# A word starts with a word character and continues with word characters and combining marks, so a mark with no base
# character is not a token. Zero-width joiners and non-joiners (U+200C, U+200D) may join two parts of one word, as in
# Sinhala conjuncts such as "ශ්‍රී". CJK characters are still one token each.
_WORD_START = rf"[^\W{_CJK_CHAR_CLASS}]"
_WORD_CHAR = rf"{_WORD_START}|[{_COMBINING_MARK_CHAR_CLASS}]"
_DEFAULT_BM25_TOKENIZATION_REGEX = (
rf"{_WORD_START}(?:{_WORD_CHAR})*(?:[\u200c\u200d](?:{_WORD_CHAR})+)*|[{_CJK_CHAR_CLASS}]"
)


@dataclass
Expand Down Expand Up @@ -108,7 +145,7 @@ class InMemoryDocumentStore:

def __init__(
self,
bm25_tokenization_regex: str = _DEFAULT_BM25_TOKENIZATION_REGEX,
bm25_tokenization_regex: str | None = _DEFAULT_BM25_TOKENIZATION_REGEX,
bm25_algorithm: Literal["BM25Okapi", "BM25L", "BM25Plus"] = "BM25L",
bm25_parameters: dict | None = None,
embedding_similarity_function: Literal["dot_product", "cosine"] = "dot_product",
Expand All @@ -125,7 +162,10 @@ def __init__(
:param bm25_tokenization_regex:
The regular expression used to tokenize the text for BM25 retrieval. The default groups word
characters into word tokens and splits Chinese, Japanese and Korean text into one token per character.
Text is lowercased and NFC-normalized before tokenization.
Text is lowercased and NFC-normalized before tokenization. Combining marks (such as the vowel signs of
Sinhala, Tamil or Devanagari) stay part of the word. `None` also selects the default. The default is
serialized as `None`, because it is built from the running Python's Unicode data and is several kilobytes
long.
:param bm25_algorithm: The BM25 algorithm to use. One of "BM25Okapi", "BM25L", or "BM25Plus".
:param bm25_parameters: Parameters for BM25 implementation in a dictionary format.
For example: `{'k1':1.5, 'b':0.75, 'epsilon':0.25}`
Expand All @@ -148,6 +188,8 @@ def __init__(
If `False` (the default), the timezone from the aware datetime is copied to the naive one before
comparing.
"""
if bm25_tokenization_regex is None:
bm25_tokenization_regex = _DEFAULT_BM25_TOKENIZATION_REGEX
self.bm25_tokenization_regex = bm25_tokenization_regex
self.tokenizer = re.compile(bm25_tokenization_regex).findall

Expand Down Expand Up @@ -428,7 +470,12 @@ def to_dict(self) -> dict[str, Any]:
"""
return default_to_dict(
self,
bm25_tokenization_regex=self.bm25_tokenization_regex,
# The default is stored as None: it is long and depends on the Python version's Unicode data.
bm25_tokenization_regex=(
None
if self.bm25_tokenization_regex == _DEFAULT_BM25_TOKENIZATION_REGEX
else self.bm25_tokenization_regex
),
bm25_algorithm=self.bm25_algorithm,
bm25_parameters=self.bm25_parameters,
embedding_similarity_function=self.embedding_similarity_function,
Expand Down
12 changes: 12 additions & 0 deletions releasenotes/notes/bm25-combining-marks-7c4e9a2b1d3f5e60.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
---
fixes:
- |
Fixed BM25 tokenization in ``InMemoryDocumentStore`` for scripts that write vowels and other signs as combining
marks, such as Sinhala, Tamil, Hindi and other Devanagari languages, Bengali, and fully vowelled Arabic and
Hebrew. Python's ``\w`` does not match combining marks, so the default ``bm25_tokenization_regex`` used to cut
these words into fragments (Sinhala "ජනාධිපතිවරයා" became "ජන", "ධ", "පත", "වරය"). Words now stay whole, and a
zero-width joiner inside a word, as in Sinhala "ශ්‍රී", no longer splits it.
The default ``bm25_tokenization_regex`` is now serialized as ``None`` and rebuilt when the store is loaded,
because it is several kilobytes long and built from the running Python's Unicode data. Passing ``None`` to
``InMemoryDocumentStore`` also selects the default. Pipelines saved with an older default regex keep using
that saved regex until it is removed or set to ``None``.
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@
from haystack.components.retrievers import InMemoryEmbeddingRetriever, MultiQueryEmbeddingRetriever
from haystack.components.writers import DocumentWriter
from haystack.document_stores.in_memory import InMemoryDocumentStore
from haystack.document_stores.in_memory.document_store import _DEFAULT_BM25_TOKENIZATION_REGEX
from haystack.document_stores.types import DuplicatePolicy


Expand Down Expand Up @@ -123,7 +122,7 @@ def test_to_dict(self, in_memory_doc_store):
"document_store": {
"type": "haystack.document_stores.in_memory.document_store.InMemoryDocumentStore",
"init_parameters": {
"bm25_tokenization_regex": _DEFAULT_BM25_TOKENIZATION_REGEX,
"bm25_tokenization_regex": None,
"bm25_algorithm": "BM25L",
"bm25_parameters": {},
"embedding_similarity_function": "dot_product",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@
from haystack.components.retrievers import InMemoryBM25Retriever, MultiQueryTextRetriever
from haystack.components.writers import DocumentWriter
from haystack.document_stores.in_memory import InMemoryDocumentStore
from haystack.document_stores.in_memory.document_store import _DEFAULT_BM25_TOKENIZATION_REGEX
from haystack.document_stores.types import DuplicatePolicy


Expand Down Expand Up @@ -91,7 +90,7 @@ def test_to_dict(self, in_memory_doc_store):
"document_store": {
"type": "haystack.document_stores.in_memory.document_store.InMemoryDocumentStore",
"init_parameters": {
"bm25_tokenization_regex": _DEFAULT_BM25_TOKENIZATION_REGEX,
"bm25_tokenization_regex": None,
"bm25_algorithm": "BM25L",
"bm25_parameters": {},
"embedding_similarity_function": "dot_product",
Expand Down
3 changes: 1 addition & 2 deletions test/components/retrievers/test_multi_retriever.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@
from haystack.components.retrievers.types import TextRetriever
from haystack.components.writers import DocumentWriter
from haystack.document_stores.in_memory import InMemoryDocumentStore
from haystack.document_stores.in_memory.document_store import _DEFAULT_BM25_TOKENIZATION_REGEX
from haystack.document_stores.types import DuplicatePolicy
from haystack.utils.experimental import ExperimentalWarning

Expand Down Expand Up @@ -335,7 +334,7 @@ def test_to_dict(self):
"document_store": {
"type": "haystack.document_stores.in_memory.document_store.InMemoryDocumentStore",
"init_parameters": {
"bm25_tokenization_regex": _DEFAULT_BM25_TOKENIZATION_REGEX,
"bm25_tokenization_regex": None,
"bm25_algorithm": "BM25L",
"bm25_parameters": {},
"embedding_similarity_function": "dot_product",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@
from haystack.components.retrievers import InMemoryBM25Retriever
from haystack.components.retrievers.sentence_window_retriever import SentenceWindowRetriever
from haystack.document_stores.in_memory import InMemoryDocumentStore
from haystack.document_stores.in_memory.document_store import _DEFAULT_BM25_TOKENIZATION_REGEX


class TestSentenceWindowRetriever:
Expand Down Expand Up @@ -105,7 +104,7 @@ def test_to_dict(self, in_memory_doc_store):
"init_parameters": {
"bm25_algorithm": "BM25L",
"bm25_parameters": {},
"bm25_tokenization_regex": _DEFAULT_BM25_TOKENIZATION_REGEX,
"bm25_tokenization_regex": None,
"embedding_similarity_function": "dot_product",
"index": ANY,
"shared": True,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@
from haystack.components.retrievers import InMemoryEmbeddingRetriever, TextEmbeddingRetriever
from haystack.components.writers import DocumentWriter
from haystack.document_stores.in_memory import InMemoryDocumentStore
from haystack.document_stores.in_memory.document_store import _DEFAULT_BM25_TOKENIZATION_REGEX
from haystack.document_stores.types import DuplicatePolicy


Expand Down Expand Up @@ -115,7 +114,7 @@ def test_to_dict(self):
"document_store": {
"type": "haystack.document_stores.in_memory.document_store.InMemoryDocumentStore",
"init_parameters": {
"bm25_tokenization_regex": _DEFAULT_BM25_TOKENIZATION_REGEX,
"bm25_tokenization_regex": None,
"bm25_algorithm": "BM25L",
"bm25_parameters": {},
"embedding_similarity_function": "dot_product",
Expand Down
56 changes: 55 additions & 1 deletion test/document_stores/test_in_memory.py
Original file line number Diff line number Diff line change
Expand Up @@ -441,7 +441,7 @@ def test_to_dict(self, in_memory_doc_store):
assert data == {
"type": "haystack.document_stores.in_memory.document_store.InMemoryDocumentStore",
"init_parameters": {
"bm25_tokenization_regex": _DEFAULT_BM25_TOKENIZATION_REGEX,
"bm25_tokenization_regex": None,
"bm25_algorithm": "BM25L",
"bm25_parameters": {},
"embedding_similarity_function": "dot_product",
Expand Down Expand Up @@ -477,6 +477,21 @@ def test_to_dict_with_custom_init_parameters(self):
},
}

def test_to_dict_from_dict_round_trip_keeps_default_tokenizer(self):
# The default regex is long and depends on the Python version's Unicode data, so it is saved as None and
# rebuilt on load.
store = InMemoryDocumentStore()
data = store.to_dict()
assert data["init_parameters"]["bm25_tokenization_regex"] is None

loaded = InMemoryDocumentStore.from_dict(data)
assert loaded.bm25_tokenization_regex == _DEFAULT_BM25_TOKENIZATION_REGEX
assert loaded._tokenize_bm25("ජනාධිපතිවරයා ගියේය") == ["ජනාධිපතිවරයා", "ගියේය"]

def test_init_with_none_regex_uses_default(self):
store = InMemoryDocumentStore(bm25_tokenization_regex=None)
assert store.bm25_tokenization_regex == _DEFAULT_BM25_TOKENIZATION_REGEX

@patch("haystack.document_stores.in_memory.document_store.re")
def test_from_dict(self, mock_regex):
data = {
Expand Down Expand Up @@ -1235,6 +1250,45 @@ def test_bm25_tokenization_handles_cjk_details(self, in_memory_doc_store):
assert nfd != nfc
assert tokenize(nfd) == tokenize(nfc) == ["\uc11c", "\uc6b8", "\uc740"]

def test_bm25_tokenization_keeps_combining_marks(self, in_memory_doc_store):
tokenize = in_memory_doc_store._tokenize_bm25

# Vowel signs are combining marks, which `\w` does not match, so words in these scripts used to be cut
# into fragments (Sinhala "ජනාධිපතිවරයා" became "ජන", "ධ", "පත", "වරය").
sinhala = "ජනාධිපතිවරයා පාර්ලිමේන්තුවට ගියේය."
assert tokenize(sinhala) == ["ජනාධිපතිවරයා", "පාර්ලිමේන්තුවට", "ගියේය"]
assert tokenize("தமிழ் மொழி") == ["தமிழ்", "மொழி"]
assert tokenize("বাংলা ভাষা") == ["বাংলা", "ভাষা"]
assert tokenize("كَتَبَ الطَّالِبُ") == ["كَتَبَ", "الطَّالِبُ"]

# The danda (।, ॥) is punctuation, so it still separates words.
hindi = "भारत की राजधानी नई दिल्ली है। ठीक है॥"
assert tokenize(hindi) == ["भारत", "की", "राजधानी", "नई", "दिल्ली", "है", "ठीक", "है"]

# A zero-width joiner inside a word (Sinhala "ශ්‍රී") keeps the word whole, and is not a token alone.
assert tokenize("ශ්\u200dරී ලංකාව") == ["ශ්\u200dරී", "ලංකාව"]
assert tokenize("a \u200d b") == ["a", "b"]

# Decomposed Latin letters stay whole too.
assert tokenize("cafe\u0301") == ["café"]
# A combining mark with no base character is not a token on its own.
assert tokenize("a \u0301 b") == ["a", "b"]

def test_bm25_retrieval_with_combining_mark_script_query(self, in_memory_doc_store):
# With words cut into fragments, the Tamil query "தமிழ்" (Tamil) also matched "தம்பி" (younger brother)
# and "மகிழ்ச்சியாக" (happily), which only share fragments with it. Only the document with the word
# itself should match.
tamil = "தமிழ் மொழி இலங்கையில் பேசப்படுகிறது."
docs = [
Document(content=tamil),
Document(content="தம்பி மழையில் நனைந்தான்."),
Document(content="அவன் மிகவும் மகிழ்ச்சியாக இருந்தான்."),
]
in_memory_doc_store.write_documents(docs)

results = in_memory_doc_store.bm25_retrieval(query="தமிழ்", top_k=3)
assert [doc.content for doc in results] == [tamil]

def test_bm25_retrieval_with_single_char_query(self, in_memory_doc_store):
docs = [
Document(content="C programming language"),
Expand Down
3 changes: 1 addition & 2 deletions test/tools/test_pipeline_tool.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@
from haystack.components.retrievers import InMemoryBM25Retriever, InMemoryEmbeddingRetriever
from haystack.dataclasses import ChatMessage
from haystack.document_stores.in_memory import InMemoryDocumentStore
from haystack.document_stores.in_memory.document_store import _DEFAULT_BM25_TOKENIZATION_REGEX
from haystack.tools import PipelineTool


Expand Down Expand Up @@ -69,7 +68,7 @@ def sample_pipeline_dict():
"document_store": {
"type": "haystack.document_stores.in_memory.document_store.InMemoryDocumentStore",
"init_parameters": {
"bm25_tokenization_regex": _DEFAULT_BM25_TOKENIZATION_REGEX,
"bm25_tokenization_regex": None,
"bm25_algorithm": "BM25L",
"bm25_parameters": {},
"embedding_similarity_function": "dot_product",
Expand Down