Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 37 additions & 1 deletion haystack/components/preprocessors/sentence_tokenizer.py
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,15 @@
# Closing brackets and quotes that can trail a sentence ending, as in `He said "Hi." Bye.` or `(Hi.) Bye.`
SENTENCE_CLOSING_CHARS_RE = r"[\)\]}\"'’”»]*"

# Full-width sentence terminators of Chinese and Japanese. These scripts put no space between sentences, so NLTK,
# which looks for whitespace after a sentence ending, never splits there. A terminator that a closing quote or
# bracket follows ends quoted or bracketed text, not the sentence, so it is not a split point. The full-width full
# stop `.` (common in Japanese technical writing) is also a decimal point and a list number, as in `3.5` and
# `1.はじめに`, so a lone `.` next to a digit is not a split point either. The terminators are in the lookahead too,
# so that a run like `?!」` is never split in the middle.
_CJK_SENTENCE_END_RE = re.compile(r"[。!?。.]+(?![。!?。.」』”’)】〕》〉\)\]}\"'])")
_DIGITS = frozenset("01234567890123456789")

if nltk_imports.is_successful():

def load_sentence_tokenizer(
Expand Down Expand Up @@ -199,12 +208,39 @@ def split_sentences(self, text: str) -> list[dict[str, Any]]:
:param text: The text to split.
:returns: list of sentences with positions.
"""
sentence_spans = list(self.sentence_tokenizer.span_tokenize(text))
sentence_spans = self._split_at_cjk_sentence_ends(text, list(self.sentence_tokenizer.span_tokenize(text)))
if self.use_split_rules:
sentence_spans = SentenceSplitter._apply_split_rules(text, sentence_spans)

return [{"sentence": text[start:end], "start": start, "end": end} for start, end in sentence_spans]

def _split_at_cjk_sentence_ends(self, text: str, sentence_spans: list[tuple[int, int]]) -> list[tuple[int, int]]:
"""
Splits the sentence spans after the full-width sentence terminators of Chinese and Japanese (。!?.).

:param text: The text to split.
:param sentence_spans: The sentence spans found by the NLTK tokenizer.
:returns: The sentence spans, split after each full-width terminator that has more text after it.
"""
new_sentence_spans = []
for start, end in sentence_spans:
for match in _CJK_SENTENCE_END_RE.finditer(text, start, end):
if match.group() == "." and (
(match.start() > 0 and text[match.start() - 1] in _DIGITS)
or (match.end() < len(text) and text[match.end()] in _DIGITS)
):
continue
next_start = match.end()
while next_start < end and text[next_start].isspace():
next_start += 1
if next_start == end:
break
# with keep_white_spaces=True a sentence keeps the whitespace up to the next one
new_sentence_spans.append((start, next_start if self.keep_white_spaces else match.end()))
start = next_start
new_sentence_spans.append((start, end))
return new_sentence_spans

@staticmethod
def _apply_split_rules(text: str, sentence_spans: list[tuple[int, int]]) -> list[tuple[int, int]]:
"""
Expand Down
20 changes: 20 additions & 0 deletions releasenotes/notes/split-cjk-sentences-01c182ccc8885ce6.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
---
enhancements:
- |
``DocumentSplitter`` and ``RecursiveDocumentSplitter`` now split Chinese and Japanese text into sentences at the
full-width sentence terminators ``。``, ``!``, ``?`` and ``.``. Before, a Chinese or Japanese document was one
sentence, because these languages put no space after a sentence. A terminator followed by a closing quote or
bracket is not a split point, and neither is a ``.`` next to a digit, as in ``3.5``. This works with every
``language`` setting.

.. code-block:: python

from haystack import Document
from haystack.components.preprocessors import DocumentSplitter

splitter = DocumentSplitter(split_by="sentence", split_length=1)
splitter.warm_up()

result = splitter.run(documents=[Document(content="我今天去了银行。利息是多少?")])
print([doc.content for doc in result["documents"]])
# >> ["我今天去了银行。", "利息是多少?"]
41 changes: 41 additions & 0 deletions test/components/preprocessors/test_sentence_tokenizer.py
Original file line number Diff line number Diff line change
Expand Up @@ -208,6 +208,47 @@ def test_split_sentences_does_not_split_on_danda_for_english(keep_white_spaces:
assert splitter.split_sentences(text) == [{"sentence": text, "start": 0, "end": len(text)}]


@pytest.mark.parametrize("keep_white_spaces", [False, True])
def test_split_sentences_splits_on_full_width_terminators(keep_white_spaces: bool) -> None:
# Chinese and Japanese put no space between sentences
text = "我今天去了银行。利息是多少?我不知道! \n今日は銀行に行きました。わかりません。"
splitter = SentenceSplitter(language="en", keep_white_spaces=keep_white_spaces)

sentences = splitter.split_sentences(text)
expected = ["我今天去了银行。", "利息是多少?", "我不知道! \n", "今日は銀行に行きました。", "わかりません。"]
if not keep_white_spaces:
expected = [sentence.rstrip() for sentence in expected]

assert [sentence["sentence"] for sentence in sentences] == expected
for sentence in sentences:
assert text[sentence["start"] : sentence["end"]] == sentence["sentence"]
if keep_white_spaces:
assert "".join(sentence["sentence"] for sentence in sentences) == text


@pytest.mark.parametrize("keep_white_spaces", [False, True])
def test_split_sentences_does_not_split_on_a_full_width_terminator_inside_quotes(keep_white_spaces: bool) -> None:
# stacked terminators before a closing quote, as in `?!」`, must not be split in the middle either
text = "他说:“我走了。”然后离开了。「利息はいくらですか?!」と聞きました。"
splitter = SentenceSplitter(language="en", keep_white_spaces=keep_white_spaces)

sentences = splitter.split_sentences(text)

assert [sentence["sentence"] for sentence in sentences] == [
"他说:“我走了。”然后离开了。",
"「利息はいくらですか?!」と聞きました。",
]


def test_split_sentences_splits_on_full_width_full_stop_but_not_next_to_digits() -> None:
text = "1.はじめに\n結果は3.5でした.次の文."
splitter = SentenceSplitter(language="en")

sentences = splitter.split_sentences(text)

assert [sentence["sentence"] for sentence in sentences] == ["1.はじめに\n結果は3.5でした.", "次の文."]


def test_split_sentences_performance() -> None:
# make sure our regex is not vulnerable to Regex Denial of Service (ReDoS)
# https://owasp.org/www-community/attacks/Regular_expression_Denial_of_Service_-_ReDoS
Expand Down