Skip to content
Open
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions haystack/components/preprocessors/python_code_splitter.py
Original file line number Diff line number Diff line change
Expand Up @@ -350,6 +350,12 @@ def _extract_units(self, source: str) -> list[_CodeUnit]:
while node_idx < node_count:
node = body[node_idx]

# A statement that shares its line with the previous one (e.g. `x = 1; y = 2`) is already part of the
# previous unit's source, so it must not become an empty unit of its own.
if (node.end_lineno or node.lineno) < cursor:
node_idx += 1
continue

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this still breaks when the statement after the ; runs onto more lines, e.g. x = 1; y = dict(\n a=1,\n) with max_effective_lines=1 gives a chunk that ends at y = dict( and a second chunk a=1,\n) with start_line 1. letting the previous unit take the rest of those lines fixes it:

Suggested change
# A statement that shares its line with the previous one (e.g. `x = 1; y = 2`) is already part of the
# previous unit's source, so it must not become an empty unit of its own.
if (node.end_lineno or node.lineno) < cursor:
node_idx += 1
continue
# A statement that starts on the line the previous unit ended on (e.g. `x = 1; y = 2`) is already part
# of that unit's source. If it runs onto later lines, the previous unit takes those lines too.
if node.lineno < cursor:
end = node.end_lineno or node.lineno
if end >= cursor and units:
units[-1].source += self._slice_lines(source_lines, cursor, end)
units[-1].end_line = end
cursor = end + 1
node_idx += 1
continue

fixed this plus a test in this commit, you can just cherry pick it instead of redoing it: its-amann@9911d85

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks, good catch. I cherry-picked your commit as-is (59c6b38, authorship kept), so a statement after the ; that runs onto later lines now stays in the previous chunk. All 89 splitter tests pass locally, including your two new cases.


# Module docstring (only valid as the very first statement).
if (
node_idx == 0
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
---
fixes:
- |
``PythonCodeSplitter`` no longer emits an empty chunk, or a chunk with a wrong ``start_line``, when top-level
statements share a line separated by a semicolon (for example ``x = 1; y = 2``). The second statement was turned
into a unit with no content, which became an empty chunk at the end of the file or was merged into the next chunk
and moved its ``start_line`` back to the previous line. Such statements now stay in the chunk of the line they
are on.
14 changes: 14 additions & 0 deletions test/components/preprocessors/test_python_code_splitter.py
Original file line number Diff line number Diff line change
Expand Up @@ -285,6 +285,20 @@ def test_form_feed_does_not_shift_ast_line_numbers(self):
assert [(chunk.meta["start_line"], chunk.meta["end_line"]) for chunk in chunks] == [(2, 3), (5, 6)]
assert "".join(content or "" for content in contents) == source

@pytest.mark.parametrize(
"source",
["def first():\n return 1\nx = 1; y = 2\n", "import os\nx = 1; y = 2\ndef first():\n return 1\n"],
)
def test_statements_sharing_a_line_stay_in_one_chunk(self, source):
splitter = PythonCodeSplitter(min_effective_lines=1, max_effective_lines=1, preserve_class_definition=False)
chunks = splitter.run(documents=[Document(content=source)])["documents"]

assert all(chunk.content for chunk in chunks)
assert "".join(chunk.content or "" for chunk in chunks) == source
lines = source.splitlines(keepends=True)
for chunk in chunks:
assert chunk.content == "".join(lines[chunk.meta["start_line"] - 1 : chunk.meta["end_line"]])


class TestFileNamePropagation:
def test_file_name_propagated_to_all_chunks(self, simple_module_source):
Expand Down