Skip to content
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 8 additions & 2 deletions haystack/utils/filters.py
Original file line number Diff line number Diff line change
Expand Up @@ -170,12 +170,18 @@ def _greater_than(value: Any, filter_value: Any, strict_datetime_comparison: boo


def _parse_date(value: str) -> datetime:
"""Try parsing the value as an ISO format date, then fall back to dateutil.parser."""
"""
Try parsing the value as an ISO format date, then fall back to dateutil.parser.

Partial dates (e.g. "2024" or "2024-06") are filled with the start of the period (January 1st, or the 1st of the
month) instead of dateutil's default of taking the missing parts from the current date, so that filters behave
the same on every day.
Comment thread
sjrl marked this conversation as resolved.
Resolved
"""
try:
return datetime.fromisoformat(value)
except (ValueError, TypeError):
try:
return dateutil.parser.parse(value)
return dateutil.parser.parse(value, default=datetime(1, 1, 1))
Comment thread
sjrl marked this conversation as resolved.
Resolved
except (ValueError, TypeError) as exc:
msg = (
"Can't compare strings using operators '>', '>=', '<', '<='. "
Expand Down
7 changes: 7 additions & 0 deletions releasenotes/notes/fix-partial-date-filter-13338.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
---
fixes:
- |
Fix ``Filter`` range operators (``>``, ``>=``, ``<``, ``<=``) incorrectly
completing partial dates (e.g. ``"2024"``) with the current month and day,
which silently dropped matching documents. Partial dates are now completed
to the start of the period.
Comment thread
sjrl marked this conversation as resolved.
Resolved
47 changes: 46 additions & 1 deletion test/utils/test_filters.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@

from haystack import Document
from haystack.errors import FilterError
from haystack.utils.filters import document_matches_filter
from haystack.utils.filters import _parse_date, document_matches_filter

document_matches_filter_data = [
# == operator params
Expand Down Expand Up @@ -222,6 +222,31 @@
True,
id=">= operator with equal datetime in different ISO 8601 format",
),
pytest.param(
# Partial dates are resolved to the start of the period, not filled from the current date.
{"field": "meta.date", "operator": ">=", "value": "1969"},
Document(meta={"date": "1969-01-01"}),
True,
id=">= operator with partial year filter value matches start of the year",
),
pytest.param(
{"field": "meta.date", "operator": ">=", "value": "1969"},
Document(meta={"date": "1968-12-31T23:59:59"}),
False,
id=">= operator with partial year filter value excludes previous year",
Comment thread
sjrl marked this conversation as resolved.
Resolved
),
pytest.param(
{"field": "meta.date", "operator": ">=", "value": "1969-07"},
Document(meta={"date": "1969-07-21"}),
Comment thread
sjrl marked this conversation as resolved.
Resolved
True,
id=">= operator with partial year-month filter value matches start of the month",
),
pytest.param(
{"field": "meta.date", "operator": ">=", "value": "1969-07"},
Document(meta={"date": "1969-06-30"}),
False,
id=">= operator with partial year-month filter value excludes previous month",
),
pytest.param(
{"field": "meta.date", "operator": ">=", "value": "1969-07-21T20:17:40+00:00"},
Document(meta={"date": "1969-07-21T22:17:40+02:00"}),
Expand Down Expand Up @@ -277,6 +302,19 @@
False,
id="< operator with equal ISO 8601 datetime Document value",
),
pytest.param(
# Partial dates are resolved to the start of the period, not filled from the current date.
{"field": "meta.date", "operator": "<", "value": "1969"},
Document(meta={"date": "1968-06-15"}),
True,
id="< operator with partial year filter value matches previous year",
),
pytest.param(
{"field": "meta.date", "operator": "<", "value": "1969"},
Document(meta={"date": "1969-06-15"}),
False,
id="< operator with partial year filter value excludes same year",
),
pytest.param(
{"field": "meta.date", "operator": "<", "value": "1969-07-21T20:17:40"},
Document(meta={"date": "1972-12-11T19:54:58"}),
Expand Down Expand Up @@ -633,6 +671,13 @@ def test_document_matches_filter(filters, document, expected_result):
assert document_matches_filter(filters, document) == expected_result


def test_parse_date_partial_dates_resolve_to_start_of_period():
Comment thread
sjrl marked this conversation as resolved.
Resolved
# Regression test: partial dates must not be filled with the current date,
# otherwise the same filter returns different documents depending on the day it runs.
assert _parse_date("2024") == datetime(2024, 1, 1)
assert _parse_date("2024-06") == datetime(2024, 6, 1)


@pytest.mark.parametrize(
"operator,expected_result",
[
Expand Down
Loading