diff --git a/haystack/components/preprocessors/sentence_tokenizer.py b/haystack/components/preprocessors/sentence_tokenizer.py index d685222758b..206cc9eab2e 100644 --- a/haystack/components/preprocessors/sentence_tokenizer.py +++ b/haystack/components/preprocessors/sentence_tokenizer.py @@ -77,6 +77,15 @@ # Closing brackets and quotes that can trail a sentence ending, as in `He said "Hi." Bye.` or `(Hi.) Bye.` SENTENCE_CLOSING_CHARS_RE = r"[\)\]}\"'’”»]*" +# Full-width sentence terminators of Chinese and Japanese. These scripts put no space between sentences, so NLTK, +# which looks for whitespace after a sentence ending, never splits there. A terminator that a closing quote or +# bracket follows ends quoted or bracketed text, not the sentence, so it is not a split point. The full-width full +# stop `.` (common in Japanese technical writing) is also a decimal point and a list number, as in `3.5` and +# `1.はじめに`, so a lone `.` next to a digit is not a split point either. The terminators are in the lookahead too, +# so that a run like `?!」` is never split in the middle. +_CJK_SENTENCE_END_RE = re.compile(r"[。!?。.]+(?![。!?。.」』”’)】〕》〉\)\]}\"'])") +_DIGITS = frozenset("01234567890123456789") + if nltk_imports.is_successful(): def load_sentence_tokenizer( @@ -199,12 +208,39 @@ def split_sentences(self, text: str) -> list[dict[str, Any]]: :param text: The text to split. :returns: list of sentences with positions. """ - sentence_spans = list(self.sentence_tokenizer.span_tokenize(text)) + sentence_spans = self._split_at_cjk_sentence_ends(text, list(self.sentence_tokenizer.span_tokenize(text))) if self.use_split_rules: sentence_spans = SentenceSplitter._apply_split_rules(text, sentence_spans) return [{"sentence": text[start:end], "start": start, "end": end} for start, end in sentence_spans] + def _split_at_cjk_sentence_ends(self, text: str, sentence_spans: list[tuple[int, int]]) -> list[tuple[int, int]]: + """ + Splits the sentence spans after the full-width sentence terminators of Chinese and Japanese (。!?.). + + :param text: The text to split. + :param sentence_spans: The sentence spans found by the NLTK tokenizer. + :returns: The sentence spans, split after each full-width terminator that has more text after it. + """ + new_sentence_spans = [] + for start, end in sentence_spans: + for match in _CJK_SENTENCE_END_RE.finditer(text, start, end): + if match.group() == "." and ( + (match.start() > 0 and text[match.start() - 1] in _DIGITS) + or (match.end() < len(text) and text[match.end()] in _DIGITS) + ): + continue + next_start = match.end() + while next_start < end and text[next_start].isspace(): + next_start += 1 + if next_start == end: + break + # with keep_white_spaces=True a sentence keeps the whitespace up to the next one + new_sentence_spans.append((start, next_start if self.keep_white_spaces else match.end())) + start = next_start + new_sentence_spans.append((start, end)) + return new_sentence_spans + @staticmethod def _apply_split_rules(text: str, sentence_spans: list[tuple[int, int]]) -> list[tuple[int, int]]: """ diff --git a/releasenotes/notes/split-cjk-sentences-01c182ccc8885ce6.yaml b/releasenotes/notes/split-cjk-sentences-01c182ccc8885ce6.yaml new file mode 100644 index 00000000000..11b76500da8 --- /dev/null +++ b/releasenotes/notes/split-cjk-sentences-01c182ccc8885ce6.yaml @@ -0,0 +1,20 @@ +--- +enhancements: + - | + ``DocumentSplitter`` and ``RecursiveDocumentSplitter`` now split Chinese and Japanese text into sentences at the + full-width sentence terminators ``。``, ``!``, ``?`` and ``.``. Before, a Chinese or Japanese document was one + sentence, because these languages put no space after a sentence. A terminator followed by a closing quote or + bracket is not a split point, and neither is a ``.`` next to a digit, as in ``3.5``. This works with every + ``language`` setting. + + .. code-block:: python + + from haystack import Document + from haystack.components.preprocessors import DocumentSplitter + + splitter = DocumentSplitter(split_by="sentence", split_length=1) + splitter.warm_up() + + result = splitter.run(documents=[Document(content="我今天去了银行。利息是多少?")]) + print([doc.content for doc in result["documents"]]) + # >> ["我今天去了银行。", "利息是多少?"] diff --git a/test/components/preprocessors/test_sentence_tokenizer.py b/test/components/preprocessors/test_sentence_tokenizer.py index 1c86840df02..5c8bec0f46f 100644 --- a/test/components/preprocessors/test_sentence_tokenizer.py +++ b/test/components/preprocessors/test_sentence_tokenizer.py @@ -208,6 +208,47 @@ def test_split_sentences_does_not_split_on_danda_for_english(keep_white_spaces: assert splitter.split_sentences(text) == [{"sentence": text, "start": 0, "end": len(text)}] +@pytest.mark.parametrize("keep_white_spaces", [False, True]) +def test_split_sentences_splits_on_full_width_terminators(keep_white_spaces: bool) -> None: + # Chinese and Japanese put no space between sentences + text = "我今天去了银行。利息是多少?我不知道! \n今日は銀行に行きました。わかりません。" + splitter = SentenceSplitter(language="en", keep_white_spaces=keep_white_spaces) + + sentences = splitter.split_sentences(text) + expected = ["我今天去了银行。", "利息是多少?", "我不知道! \n", "今日は銀行に行きました。", "わかりません。"] + if not keep_white_spaces: + expected = [sentence.rstrip() for sentence in expected] + + assert [sentence["sentence"] for sentence in sentences] == expected + for sentence in sentences: + assert text[sentence["start"] : sentence["end"]] == sentence["sentence"] + if keep_white_spaces: + assert "".join(sentence["sentence"] for sentence in sentences) == text + + +@pytest.mark.parametrize("keep_white_spaces", [False, True]) +def test_split_sentences_does_not_split_on_a_full_width_terminator_inside_quotes(keep_white_spaces: bool) -> None: + # stacked terminators before a closing quote, as in `?!」`, must not be split in the middle either + text = "他说:“我走了。”然后离开了。「利息はいくらですか?!」と聞きました。" + splitter = SentenceSplitter(language="en", keep_white_spaces=keep_white_spaces) + + sentences = splitter.split_sentences(text) + + assert [sentence["sentence"] for sentence in sentences] == [ + "他说:“我走了。”然后离开了。", + "「利息はいくらですか?!」と聞きました。", + ] + + +def test_split_sentences_splits_on_full_width_full_stop_but_not_next_to_digits() -> None: + text = "1.はじめに\n結果は3.5でした.次の文." + splitter = SentenceSplitter(language="en") + + sentences = splitter.split_sentences(text) + + assert [sentence["sentence"] for sentence in sentences] == ["1.はじめに\n結果は3.5でした.", "次の文."] + + def test_split_sentences_performance() -> None: # make sure our regex is not vulnerable to Regex Denial of Service (ReDoS) # https://owasp.org/www-community/attacks/Regular_expression_Denial_of_Service_-_ReDoS