diff --git a/src/palace/manager/search/revision.py b/src/palace/manager/search/revision.py index ee9c9975cf..8d18e6274f 100644 --- a/src/palace/manager/search/revision.py +++ b/src/palace/manager/search/revision.py @@ -21,7 +21,7 @@ def version(self) -> int: def name_for_index(self, base_name: str) -> str: """Produce the name of the index as it will appear in Opensearch, - such as 'circulation-works-v5'.""" + such as 'circulation-works-v8'.""" return f"{base_name}-v{self.version}" def script_name(self, script_name: str) -> str: diff --git a/src/palace/manager/search/revision_directory.py b/src/palace/manager/search/revision_directory.py index 541b28897b..fa87aca16a 100644 --- a/src/palace/manager/search/revision_directory.py +++ b/src/palace/manager/search/revision_directory.py @@ -2,12 +2,9 @@ from palace.manager.core.config import CannotLoadConfiguration from palace.manager.search.revision import SearchSchemaRevision -from palace.manager.search.v5 import SearchV5 -from palace.manager.search.v6 import SearchV6 -from palace.manager.search.v7 import SearchV7 from palace.manager.search.v8 import SearchV8 -REVISIONS = [SearchV5(), SearchV6(), SearchV7(), SearchV8()] +REVISIONS = [SearchV8()] class SearchRevisionDirectory: diff --git a/src/palace/manager/search/v5.py b/src/palace/manager/search/v5.py deleted file mode 100644 index 14ac274fad..0000000000 --- a/src/palace/manager/search/v5.py +++ /dev/null @@ -1,265 +0,0 @@ -from palace.manager.search.document import ( - BASIC_TEXT, - BOOLEAN, - FILTERABLE_TEXT, - FLOAT, - INTEGER, - LONG, - SearchMappingDocument, - SearchMappingFieldType, - icu_collation_keyword, - keyword, - nested, - sort_author_keyword, -) -from palace.manager.search.revision import SearchSchemaRevision - - -class SearchV5(SearchSchemaRevision): - @property - def version(self) -> int: - return 5 - - """ - The body of this mapping looks for bibliographic information in - the core document, primarily used for matching search - requests. It also has nested documents, which are used for - filtering and ranking Works when generating other types of - feeds: - - * licensepools -- the Work has these LicensePools (includes current - availability as a boolean, but not detailed availability information) - * customlists -- the Work is on these CustomLists - * contributors -- these Contributors worked on the Work - """ - - # Use regular expressions to normalized values in sortable fields. - # These regexes are applied in order; that way "H. G. Wells" - # becomes "H G Wells" becomes "HG Wells". - CHAR_FILTERS = { - "remove_apostrophes": dict( - type="pattern_replace", - pattern="'", - replacement="", - ) - } - - AUTHOR_CHAR_FILTER_NAMES = [] - for name, pattern, replacement in [ - # The special author name "[Unknown]" should sort after everything - # else. REPLACEMENT CHARACTER is the final valid Unicode character. - ("unknown_author", r"\[Unknown\]", "\N{REPLACEMENT CHARACTER}"), - # Works by a given primary author should be secondarily sorted - # by title, not by the other contributors. - ("primary_author_only", r"\s+;.*", ""), - # Remove parentheticals (e.g. the full name of someone who - # goes by initials). - ("strip_parentheticals", r"\s+\([^)]+\)", ""), - # Remove periods from consideration. - ("strip_periods", r"\.", ""), - # Collapse spaces for people whose sort names end with initials. - ("collapse_three_initials", r" ([A-Z]) ([A-Z]) ([A-Z])$", " $1$2$3"), - ("collapse_two_initials", r" ([A-Z]) ([A-Z])$", " $1$2"), - ]: - normalizer = dict( - type="pattern_replace", pattern=pattern, replacement=replacement - ) - CHAR_FILTERS[name] = normalizer - AUTHOR_CHAR_FILTER_NAMES.append(name) - - def __init__(self) -> None: - super().__init__() - - self._normalizers = {} - self._char_filters = {} - self._filters = {} - self._analyzers = {} - - # Set up character filters. - # - self._char_filters = self.CHAR_FILTERS - - # This normalizer is used on freeform strings that - # will be used as tokens in filters. This way we can, - # e.g. ignore capitalization when considering whether - # two books belong to the same series or whether two - # author names are the same. - self._normalizers["filterable_string"] = dict( - type="custom", filter=["lowercase", "asciifolding"] - ) - - # Set up analyzers. - # - - # We use three analyzers: - # - # 1. An analyzer based on Opensearch's default English - # analyzer, with a normal stemmer -- used as the default - # view of a text field such as 'description'. - # - # 2. An analyzer that's exactly the same as #1 but with a less - # aggressive stemmer -- used as the 'minimal' view of a - # text field such as 'description.minimal'. - # - # 3. An analyzer that's exactly the same as #2 but with - # English stopwords left in place instead of filtered out -- - # used as the 'with_stopwords' view of a text field such as - # 'title.with_stopwords'. - # - # The analyzers are identical except for the end of the filter - # chain. - # - # All three analyzers are based on Opensearch's default English - # analyzer, defined here: - # https://www.elastic.co/guide/en/elasticsearch/reference/current/analysis-lang-analyzer.html#english-analyzer - - # First, recreate the filters from the default English - # analyzer. We'll be using these to build our own analyzers. - - # Filter out English stopwords. - self._filters["english_stop"] = dict(type="stop", stopwords=["_english_"]) - # The default English stemmer, used in the en_default analyzer. - self._filters["english_stemmer"] = dict(type="stemmer", language="english") - # A less aggressive English stemmer, used in the en_minimal analyzer. - self._filters["minimal_english_stemmer"] = dict( - type="stemmer", language="minimal_english" - ) - # A filter that removes English posessives such as "'s" - self._filters["english_posessive_stemmer"] = dict( - type="stemmer", language="possessive_english" - ) - - # Some potentially useful filters that are currently not used: - # - # * keyword_marker -- Exempt certain keywords from stemming - # * synonym -- Introduce synonyms for words - # (but probably better to use synonym_graph during the search - # -- it's more flexible). - - # Here's the common analyzer configuration. The comment NEW - # means this is something we added on top of Opensearch's - # default configuration for the English analyzer. - common_text_analyzer = dict( - type="custom", - char_filter=["html_strip", "remove_apostrophes"], # NEW - tokenizer="standard", - ) - common_filter = [ - "lowercase", - "asciifolding", # NEW - ] - - # The default_text_analyzer uses Opensearch's standard - # English stemmer and removes stopwords. - self._analyzers["en_default_text_analyzer"] = dict(common_text_analyzer) - self._analyzers["en_default_text_analyzer"]["filter"] = common_filter + [ - "english_stop", - "english_stemmer", - ] - - # The minimal_text_analyzer uses a less aggressive English - # stemmer, and removes stopwords. - self._analyzers["en_minimal_text_analyzer"] = dict(common_text_analyzer) - self._analyzers["en_minimal_text_analyzer"]["filter"] = common_filter + [ - "english_stop", - "minimal_english_stemmer", - ] - - # The en_with_stopwords_text_analyzer uses the less aggressive - # stemmer and does not remove stopwords. - self._analyzers["en_with_stopwords_text_analyzer"] = dict(common_text_analyzer) - self._analyzers["en_with_stopwords_text_analyzer"]["filter"] = common_filter + [ - "minimal_english_stemmer" - ] - - # Now we need to define a special analyzer used only by the - # 'sort_author' property. - - # Here's a special filter used only by that analyzer. It - # duplicates the filter used by the icu_collation_keyword data - # type. - self._filters["en_sortable_filter"] = dict( - type="icu_collation", language="en", country="US" - ) - - # Here's the analyzer used by the 'sort_author' property. - # It's the same as icu_collation_keyword, but it has some - # extra character filters -- regexes that do things like - # convert "Tolkien, J. R. R." to "Tolkien, JRR". - # - # This is necessary because normal icu_collation_keyword - # fields can't specify char_filter. - self._analyzers["en_sort_author_analyzer"] = dict( - tokenizer="keyword", - filter=["en_sortable_filter"], - char_filter=self.AUTHOR_CHAR_FILTER_NAMES, - ) - - self._fields: dict[str, SearchMappingFieldType] = { - "summary": BASIC_TEXT, - "title": FILTERABLE_TEXT, - "subtitle": FILTERABLE_TEXT, - "series": FILTERABLE_TEXT, - "classifications.term": FILTERABLE_TEXT, - "author": FILTERABLE_TEXT, - "publisher": FILTERABLE_TEXT, - "imprint": FILTERABLE_TEXT, - "presentation_ready": BOOLEAN, - "sort_title": icu_collation_keyword(), - "sort_author": sort_author_keyword(), - "series_position": INTEGER, - "work_id": INTEGER, - "last_update_time": LONG, - "published": LONG, - "audience": keyword(), - "language": keyword(), - } - - contributors = nested() - contributors.add_property("display_name", FILTERABLE_TEXT) - contributors.add_property("sort_name", FILTERABLE_TEXT) - contributors.add_property("family_name", FILTERABLE_TEXT) - contributors.add_property("role", keyword()) - contributors.add_property("lc", keyword()) - contributors.add_property("viaf", keyword()) - self._fields["contributors"] = contributors - - licensepools = nested() - licensepools.add_property("collection_id", INTEGER) - licensepools.add_property("data_source_id", INTEGER) - licensepools.add_property("availability_time", LONG) - licensepools.add_property("available", BOOLEAN) - licensepools.add_property("open_access", BOOLEAN) - licensepools.add_property("suppressed", BOOLEAN) - licensepools.add_property("licensed", BOOLEAN) - licensepools.add_property("medium", keyword()) - self._fields["licensepools"] = licensepools - - identifiers = nested() - identifiers.add_property("type", keyword()) - identifiers.add_property("identifier", keyword()) - self._fields["identifiers"] = identifiers - - genres = nested() - genres.add_property("scheme", keyword()) - genres.add_property("name", keyword()) - genres.add_property("term", keyword()) - genres.add_property("weight", FLOAT) - self._fields["genres"] = genres - - customlists = nested() - customlists.add_property("list_id", INTEGER) - customlists.add_property("first_appearance", LONG) - customlists.add_property("featured", BOOLEAN) - self._fields["customlists"] = customlists - - def mapping_document(self) -> SearchMappingDocument: - document = SearchMappingDocument() - document.settings["analysis"] = dict( - filter=dict(self._filters), - char_filter=dict(self._char_filters), - normalizer=dict(self._normalizers), - analyzer=dict(self._analyzers), - ) - document.properties = self._fields - return document diff --git a/src/palace/manager/search/v6.py b/src/palace/manager/search/v6.py deleted file mode 100644 index 7c613446f6..0000000000 --- a/src/palace/manager/search/v6.py +++ /dev/null @@ -1,14 +0,0 @@ -from palace.manager.search.document import ( - INTEGER, -) -from palace.manager.search.v5 import SearchV5 - - -class SearchV6(SearchV5): - @property - def version(self) -> int: - return 6 - - def __init__(self) -> None: - super().__init__() - self._fields["lane_priority_level"] = INTEGER diff --git a/src/palace/manager/search/v7.py b/src/palace/manager/search/v7.py deleted file mode 100644 index dd6fbc802e..0000000000 --- a/src/palace/manager/search/v7.py +++ /dev/null @@ -1,15 +0,0 @@ -from typing import cast - -from palace.manager.search.document import LONG, SearchMappingFieldTypeObject -from palace.manager.search.v6 import SearchV6 - - -class SearchV7(SearchV6): - @property - def version(self) -> int: - return 7 - - def __init__(self) -> None: - super().__init__() - licensepools = cast(SearchMappingFieldTypeObject, self._fields["licensepools"]) - licensepools.add_property("last_updated", LONG) diff --git a/src/palace/manager/search/v8.py b/src/palace/manager/search/v8.py index 2b231f1e06..d586572297 100644 --- a/src/palace/manager/search/v8.py +++ b/src/palace/manager/search/v8.py @@ -20,15 +20,10 @@ class SearchV8(SearchSchemaRevision): This revision is intentionally **self-contained**: it does not inherit from any previous revision and defines its complete mapping (analyzers, filters, - and fields) directly. This is how new schema revisions are meant to be defined. + and fields) directly. This is how new schema revisions are to be defined. - We formed a bad habit for several old schema revisions where they just chained - off one another (``SearchV7`` -> ``SearchV6`` -> ``SearchV5``) which meant - no old revision could be removed without breaking the ones build on top of - it. - - From now on every revision should stand alone so that, once nothing in - production is using an older version, that version's module can simply be deleted. + Every search revision should stand alone so that, once nothing in production + is using an older version, that version's module can simply be deleted. Changes in v8: - The shard and replica counts are set explicitly. diff --git a/tests/manager/search/test_external_search.py b/tests/manager/search/test_external_search.py index f585ddaabb..4ad1841d0e 100644 --- a/tests/manager/search/test_external_search.py +++ b/tests/manager/search/test_external_search.py @@ -1,4 +1,3 @@ -import re from collections.abc import Callable import pytest @@ -18,7 +17,6 @@ ) from palace.manager.search.filter import Filter from palace.manager.search.pagination import Pagination, SortKeyPagination -from palace.manager.search.v5 import SearchV5 from palace.manager.sqlalchemy.model.classification import Genre from palace.manager.sqlalchemy.model.collection import Collection from palace.manager.sqlalchemy.model.customlist import CustomList @@ -169,54 +167,6 @@ def test_clear_search_documents( end_to_end_search_fixture.expect_results([], "") -class TestSearchV5: - def test_character_filters(self): - # Verify the functionality of the regular expressions we tell - # Opensearch to use when normalizing fields that will be used - # for searching. - filters = [] - for filter_name in SearchV5.AUTHOR_CHAR_FILTER_NAMES: - configuration = SearchV5.CHAR_FILTERS[filter_name] - find = re.compile(configuration["pattern"]) - replace = configuration["replacement"] - # Hack to (imperfectly) convert Java regex format to Python format. - # $1 -> \1 - replace = replace.replace("$", "\\") - filters.append((find, replace)) - - def filters_to(start, finish): - """When all the filters are applied to `start`, - the result is `finish`. - """ - for find, replace in filters: - start = find.sub(replace, start) - assert start == finish - - # Only the primary author is considered for sorting purposes. - filters_to("Adams, John Joseph ; Yu, Charles", "Adams, John Joseph") - - # The special system author '[Unknown]' is replaced with - # REPLACEMENT CHARACTER so it will be last in sorted lists. - filters_to("[Unknown]", "\N{REPLACEMENT CHARACTER}") - - # Periods are removed. - filters_to("Tepper, Sheri S.", "Tepper, Sheri S") - filters_to("Tepper, Sheri S", "Tepper, Sheri S") - - # The initials of authors who go by initials are normalized - # so that their books all sort together. - filters_to("Wells, HG", "Wells, HG") - filters_to("Wells, H G", "Wells, HG") - filters_to("Wells, H.G.", "Wells, HG") - filters_to("Wells, H. G.", "Wells, HG") - - # It works with up to three initials. - filters_to("Tolkien, J. R. R.", "Tolkien, JRR") - - # Parentheticals are removed. - filters_to("Wells, H. G. (Herbert George)", "Wells, HG") - - class TestExternalSearchWithWorksData: adult_work: Work age_2_10: Work diff --git a/tests/manager/search/test_search_v6.py b/tests/manager/search/test_search_v6.py deleted file mode 100644 index 9ae0d11abc..0000000000 --- a/tests/manager/search/test_search_v6.py +++ /dev/null @@ -1,8 +0,0 @@ -from palace.manager.search.document import INTEGER -from palace.manager.search.v6 import SearchV6 - - -class TestSearchV6: - def test(self): - v6 = SearchV6() - assert v6._fields["lane_priority_level"] == INTEGER diff --git a/tests/manager/search/test_search_v7.py b/tests/manager/search/test_search_v7.py deleted file mode 100644 index 611d5d2ef6..0000000000 --- a/tests/manager/search/test_search_v7.py +++ /dev/null @@ -1,9 +0,0 @@ -from palace.manager.search.document import LONG -from palace.manager.search.v7 import SearchV7 - - -class TestSearchV7: - def test(self): - v7 = SearchV7() - licensepools = v7._fields["licensepools"] - assert licensepools.properties["last_updated"] == LONG diff --git a/tests/manager/search/test_search_v8.py b/tests/manager/search/test_search_v8.py index a2013627d5..8ca6c554dc 100644 --- a/tests/manager/search/test_search_v8.py +++ b/tests/manager/search/test_search_v8.py @@ -1,5 +1,6 @@ +import re + from palace.manager.search.revision import SearchSchemaRevision -from palace.manager.search.v7 import SearchV7 from palace.manager.search.v8 import SearchV8 @@ -25,15 +26,48 @@ def test_pins_index_settings(self): ]: assert setting in index - def test_mapping_matches_v7(self): - """v8 is a faithful, self-contained copy of the v7 schema: same fields - and same analysis settings. Only the index settings differ (v8 pins the - shard count).""" - v8_document = SearchV8().mapping_document() - v7_document = SearchV7().mapping_document() - - assert v8_document.serialize_properties() == v7_document.serialize_properties() - assert v8_document.settings["analysis"] == v7_document.settings["analysis"] - # v7 left the shard count to an inherited default; v8 sets it explicitly. - assert "index" not in v7_document.settings - assert "index" in v8_document.settings + def test_character_filters(self): + # Verify the functionality of the regular expressions we tell + # Opensearch to use when normalizing fields that will be used + # for searching. + filters = [] + for filter_name in SearchV8.AUTHOR_CHAR_FILTER_NAMES: + configuration = SearchV8.CHAR_FILTERS[filter_name] + find = re.compile(configuration["pattern"]) + replace = configuration["replacement"] + # Hack to (imperfectly) convert Java regex format to Python format. + # $1 -> \1 + replace = replace.replace("$", "\\") + filters.append((find, replace)) + + def filters_to(start, finish): + """When all the filters are applied to `start`, + the result is `finish`. + """ + for find, replace in filters: + start = find.sub(replace, start) + assert start == finish + + # Only the primary author is considered for sorting purposes. + filters_to("Adams, John Joseph ; Yu, Charles", "Adams, John Joseph") + + # The special system author '[Unknown]' is replaced with + # REPLACEMENT CHARACTER so it will be last in sorted lists. + filters_to("[Unknown]", "\N{REPLACEMENT CHARACTER}") + + # Periods are removed. + filters_to("Tepper, Sheri S.", "Tepper, Sheri S") + filters_to("Tepper, Sheri S", "Tepper, Sheri S") + + # The initials of authors who go by initials are normalized + # so that their books all sort together. + filters_to("Wells, HG", "Wells, HG") + filters_to("Wells, H G", "Wells, HG") + filters_to("Wells, H.G.", "Wells, HG") + filters_to("Wells, H. G.", "Wells, HG") + + # It works with up to three initials. + filters_to("Tolkien, J. R. R.", "Tolkien, JRR") + + # Parentheticals are removed. + filters_to("Wells, H. G. (Herbert George)", "Wells, HG") diff --git a/tests/mocks/search.py b/tests/mocks/search.py index 367f59ca58..ccbe1580c9 100644 --- a/tests/mocks/search.py +++ b/tests/mocks/search.py @@ -17,7 +17,7 @@ SearchService, SearchServiceFailedDocument, ) -from palace.manager.search.v5 import SearchV5 +from palace.manager.search.v8 import SearchV8 from palace.manager.sqlalchemy.model.work import Work @@ -280,4 +280,4 @@ def mapping_document(self) -> SearchMappingDocument: class MockSearchSchemaRevisionLatest(MockSearchSchemaRevision): def __init__(self, version: int) -> None: super().__init__(version) - self._document = SearchV5().mapping_document() + self._document = SearchV8().mapping_document()