diff --git a/src/SIL.LCModel.Core/Text/StringSearcher.cs b/src/SIL.LCModel.Core/Text/StringSearcher.cs
index 6c2180da..07e4816f 100644
--- a/src/SIL.LCModel.Core/Text/StringSearcher.cs
+++ b/src/SIL.LCModel.Core/Text/StringSearcher.cs
@@ -4,6 +4,7 @@
using System;
using System.Collections.Generic;
+using System.Globalization;
using System.Linq;
using System.Text;
using Icu;
@@ -30,7 +31,11 @@ public enum SearchType
///
/// Matches any words in a string.
///
- FullText
+ FullText,
+ ///
+ /// Matches any portion within a string.
+ ///
+ Substring
}
///
@@ -119,7 +124,31 @@ public IEnumerable GetItems(byte[] lower, byte[] upper)
#endregion SortKeyIndex class
+ #region SubstringEntry struct
+
+ ///
+ /// Pairs an indexed item with the raw text scanned for substring matches. Used by
+ /// .
+ ///
+ private struct SubstringEntry
+ {
+ private readonly T m_item;
+ private readonly string m_text;
+
+ public SubstringEntry(T item, string text)
+ {
+ m_item = item;
+ m_text = text;
+ }
+
+ public T Item { get { return m_item; } }
+ public string Text { get { return m_text; } }
+ }
+
+ #endregion SubstringEntry struct
+
private readonly Dictionary, SortKeyIndex> m_indices = new Dictionary, SortKeyIndex>();
+ private readonly Dictionary, List> m_rawIndices = new Dictionary, List>();
private readonly SearchType m_type;
private readonly Func m_sortKeySelector;
private readonly Func> m_tokenizer;
@@ -183,18 +212,27 @@ public void Add(T item, int indexId, ITsString tss)
///
public void Add(T item, int indexId, int wsId, string text)
{
- SortKeyIndex index = GetIndex(indexId, wsId);
+ if (string.IsNullOrEmpty(text))
+ return;
+
switch (m_type)
{
case SearchType.Exact:
case SearchType.Prefix:
- index.Add(m_sortKeySelector(wsId, text), item);
+ GetIndex(indexId, wsId).Add(m_sortKeySelector(wsId, text), item);
break;
case SearchType.FullText:
+ {
+ SortKeyIndex index = GetIndex(indexId, wsId);
foreach (string token in RemoveWhitespaceAndPunctTokens(m_tokenizer(wsId, text)))
index.Add(m_sortKeySelector(wsId, token), item);
break;
+ }
+
+ case SearchType.Substring:
+ GetRawIndex(indexId, wsId).Add(new SubstringEntry(item, text));
+ break;
}
}
@@ -233,12 +271,12 @@ public IEnumerable Search(int indexId, int wsId, string text)
if (string.IsNullOrEmpty(text))
return Enumerable.Empty();
- SortKeyIndex index = GetIndex(indexId, wsId);
switch (m_type)
{
case SearchType.Exact:
case SearchType.Prefix:
{
+ SortKeyIndex index = GetIndex(indexId, wsId);
byte[] sortKey = m_sortKeySelector(wsId, text);
var lower = new byte[text.Length * SortKeyFactor];
Collator.GetSortKeyBound(sortKey, UColBoundMode.UCOL_BOUND_LOWER, ref lower);
@@ -252,22 +290,40 @@ public IEnumerable Search(int indexId, int wsId, string text)
}
case SearchType.FullText:
- IEnumerable results = null;
- string[] tokens = RemoveWhitespaceAndPunctTokens(m_tokenizer(wsId, text)).ToArray();
- for (int i = 0; i < tokens.Length; i++)
{
- byte[] sortKey = m_sortKeySelector(wsId, tokens[i]);
- var lower = new byte[tokens[i].Length*SortKeyFactor];
- Collator.GetSortKeyBound(sortKey, UColBoundMode.UCOL_BOUND_LOWER, ref lower);
- var upper = new byte[tokens[i].Length*SortKeyFactor];
- Collator.GetSortKeyBound(sortKey,
- i < tokens.Length - 1
- ? UColBoundMode.UCOL_BOUND_UPPER
- : UColBoundMode.UCOL_BOUND_UPPER_LONG, ref upper);
- IEnumerable items = index.GetItems(lower, upper);
- results = results == null ? items : results.Intersect(items);
+ SortKeyIndex index = GetIndex(indexId, wsId);
+ IEnumerable results = null;
+ string[] tokens = RemoveWhitespaceAndPunctTokens(m_tokenizer(wsId, text)).ToArray();
+ for (int i = 0; i < tokens.Length; i++)
+ {
+ byte[] sortKey = m_sortKeySelector(wsId, tokens[i]);
+ var lower = new byte[tokens[i].Length*SortKeyFactor];
+ Collator.GetSortKeyBound(sortKey, UColBoundMode.UCOL_BOUND_LOWER, ref lower);
+ var upper = new byte[tokens[i].Length*SortKeyFactor];
+ Collator.GetSortKeyBound(sortKey,
+ i < tokens.Length - 1
+ ? UColBoundMode.UCOL_BOUND_UPPER
+ : UColBoundMode.UCOL_BOUND_UPPER_LONG, ref upper);
+ IEnumerable items = index.GetItems(lower, upper);
+ results = results == null ? items : results.Intersect(items);
+ }
+ return results;
+ }
+
+ case SearchType.Substring:
+ {
+ List raw;
+ if (!m_rawIndices.TryGetValue(Tuple.Create(indexId, wsId), out raw))
+ return Enumerable.Empty();
+ CompareInfo ci = CultureInfo.InvariantCulture.CompareInfo;
+ // Fold diacritics only when the search term itself has none: an unmarked query
+ // matches accented text ("cafe" finds "café"), but a query that includes an accent
+ // is treated as specific ("café" does not match a bare "cafe").
+ CompareOptions options = ContainsDiacritic(text)
+ ? CompareOptions.IgnoreCase
+ : CompareOptions.IgnoreCase | CompareOptions.IgnoreNonSpace;
+ return raw.Where(entry => ci.IndexOf(entry.Text, text, options) >= 0).Select(entry => entry.Item);
}
- return results;
}
return Enumerable.Empty();
@@ -278,12 +334,22 @@ private static IEnumerable RemoveWhitespaceAndPunctTokens(IEnumerable !t.All(c => Character.IsSpace(c) || Character.IsPunct(c)));
}
+ ///
+ /// True if the string contains a diacritic.
+ ///
+ private static bool ContainsDiacritic(string value)
+ {
+ return value.Normalize(NormalizationForm.FormD)
+ .Any(ch => Character.GetCharType(ch) == Character.UCharCategory.NON_SPACING_MARK);
+ }
+
///
/// Clears all of the indices.
///
public void Clear()
{
m_indices.Clear();
+ m_rawIndices.Clear();
}
private SortKeyIndex GetIndex(int indexId, int ws)
@@ -299,6 +365,18 @@ private SortKeyIndex GetIndex(int indexId, int ws)
return index;
}
+ private List GetRawIndex(int indexId, int ws)
+ {
+ var key = Tuple.Create(indexId, ws);
+ List list;
+ if (!m_rawIndices.TryGetValue(key, out list))
+ {
+ list = new List();
+ m_rawIndices[key] = list;
+ }
+ return list;
+ }
+
private static IEnumerable> GetWsStrings(ITsString tss)
{
var sb = new StringBuilder();
diff --git a/tests/SIL.LCModel.Core.Tests/Text/StringSearcherTests.cs b/tests/SIL.LCModel.Core.Tests/Text/StringSearcherTests.cs
index 15e653ea..220bb280 100644
--- a/tests/SIL.LCModel.Core.Tests/Text/StringSearcherTests.cs
+++ b/tests/SIL.LCModel.Core.Tests/Text/StringSearcherTests.cs
@@ -85,12 +85,13 @@ public void PrefixSearchTest()
}
///
- /// Tests prefix matching.
+ /// Builds the shared multi-writing-system corpus used by both
+ /// and . Item 2 deliberately mixes a
+ /// French run and an English run.
///
- [Test]
- public void FullTextSearchTest()
+ private StringSearcher BuildMultiRunCorpus(SearchType type)
{
- var searcher = new StringSearcher(SearchType.FullText, m_wsManager);
+ var searcher = new StringSearcher(type, m_wsManager);
searcher.Add(0, 0, TsStringUtils.MakeString("test", m_enWs));
searcher.Add(1, 0, TsStringUtils.MakeString("c'est une phrase", m_frWs));
ITsIncStrBldr tisb = TsStringUtils.MakeIncStrBldr();
@@ -100,11 +101,172 @@ public void FullTextSearchTest()
tisb.Append("We use it for testing purposes.");
searcher.Add(2, 0, tisb.GetString());
searcher.Add(3, 0, TsStringUtils.MakeString("Hello, how are you doing? I am doing fine. That is good to know.", m_enWs));
+ return searcher;
+ }
+
+ ///
+ /// The queries exercised by , so the substring-superset test
+ /// covers exactly the same scenarios. These are all single tokens or contiguous, in-order
+ /// phrases, and that is deliberate: substring is a superset of full-text ONLY for those shapes
+ /// (full-text ANDs word tokens regardless of order, while substring needs the whole query to
+ /// appear contiguously). Adding an out-of-order multi-word query here would make
+ /// fail; that boundary is demonstrated
+ /// by .
+ ///
+ private ITsString[] FullTextQueries()
+ {
+ return new[]
+ {
+ TsStringUtils.MakeString("test", m_enWs),
+ TsStringUtils.MakeString("c'est une", m_frWs),
+ TsStringUtils.MakeString("t", m_enWs),
+ TsStringUtils.MakeString("testing purpose", m_enWs)
+ };
+ }
+
+ ///
+ /// Tests full-text (word/prefix) matching.
+ ///
+ [Test]
+ public void FullTextSearchTest()
+ {
+ var searcher = BuildMultiRunCorpus(SearchType.FullText);
CheckSearch(searcher, TsStringUtils.MakeString("test", m_enWs), new[] {0, 2});
CheckSearch(searcher, TsStringUtils.MakeString("c'est une", m_frWs), new[] {1, 2});
CheckSearch(searcher, TsStringUtils.MakeString("t", m_enWs), new[] {0, 2, 3});
CheckSearch(searcher, TsStringUtils.MakeString("testing purpose", m_enWs), new[] {2});
}
+
+ ///
+ /// Tests substring (match-anywhere) matching, including infix, case- and diacritic-insensitivity.
+ ///
+ [Test]
+ public void SubstringSearchTest()
+ {
+ var searcher = new StringSearcher(SearchType.Substring, m_wsManager);
+ searcher.Add(0, 0, TsStringUtils.MakeString("language", m_enWs));
+ searcher.Add(1, 0, TsStringUtils.MakeString("gauge", m_enWs));
+ searcher.Add(2, 0, TsStringUtils.MakeString("résumé", m_frWs));
+ searcher.Add(3, 0, TsStringUtils.MakeString("zebra", m_enWs));
+
+ // infix match: "uage" is not a prefix of "language" but is a substring (fails under Prefix/FullText).
+ CheckSearch(searcher, TsStringUtils.MakeString("uage", m_enWs), new[] {0});
+ // interior substring
+ CheckSearch(searcher, TsStringUtils.MakeString("gua", m_enWs), new[] {0});
+ CheckSearch(searcher, TsStringUtils.MakeString("aug", m_enWs), new[] {1});
+ // case-insensitive
+ CheckSearch(searcher, TsStringUtils.MakeString("LANG", m_enWs), new[] {0});
+ // diacritic-insensitive
+ CheckSearch(searcher, TsStringUtils.MakeString("resume", m_frWs), new[] {2});
+ // whole-string still matches
+ CheckSearch(searcher, TsStringUtils.MakeString("zebra", m_enWs), new[] {3});
+ // no match anywhere
+ CheckNoResultsSearch(searcher, TsStringUtils.MakeString("xyz", m_enWs));
+ }
+
+ ///
+ /// Substring search must not miss anything a full-text search would find on the same corpus and
+ /// queries: its result set is a near superset
+ /// (see )
+ /// of the full-text result set. This guards the promise that switching Find Lexical Entry to
+ /// substring never drops a result that used to appear.
+ /// (This is a superset, not equality: substring also returns extra infix matches.)
+ ///
+ [Test]
+ public void SubstringResultsIncludeAllFullTextResults()
+ {
+ var fullText = BuildMultiRunCorpus(SearchType.FullText);
+ var substring = BuildMultiRunCorpus(SearchType.Substring);
+
+ foreach (ITsString query in FullTextQueries())
+ {
+ // StringSearcher.Search can return the same item several times (once per matching word);
+ // the real consumer (SearchEngine) dedupes via a HashSet, so compare as sets here too.
+ int[] fullTextResults = fullText.Search(0, query).Distinct().ToArray();
+ Assert.That(fullTextResults, Is.Not.Empty,
+ "query '" + query.Text + "' should match something under full-text (otherwise the check is vacuous)");
+ Assert.That(substring.Search(0, query).Distinct(), Is.SupersetOf(fullTextResults),
+ "substring dropped a full-text match for query '" + query.Text + "'");
+ }
+ }
+
+ ///
+ /// Pins the boundary of the superset guarantee: it holds only for single-token or contiguous,
+ /// in-order queries. A multi-word query whose words appear OUT OF ORDER matches under full-text
+ /// (which ANDs the word tokens regardless of order) but NOT under substring (which needs the
+ /// whole query to appear contiguously). This is the concrete case behind the scoping note on
+ /// .
+ ///
+ [Test]
+ public void Substring_isNotASupersetForOutOfOrderMultiWordQueries()
+ {
+ var fullText = new StringSearcher(SearchType.FullText, m_wsManager);
+ var substring = new StringSearcher(SearchType.Substring, m_wsManager);
+ ITsString text = TsStringUtils.MakeString("alpha beta gamma", m_enWs);
+ fullText.Add(0, 0, text);
+ substring.Add(0, 0, text);
+
+ // Words present but in a different order than the text.
+ ITsString outOfOrder = TsStringUtils.MakeString("gamma alpha", m_enWs);
+
+ Assert.That(fullText.Search(0, outOfOrder), Does.Contain(0),
+ "full-text ANDs the word tokens, so it matches the words in any order");
+ Assert.That(substring.Search(0, outOfOrder), Does.Not.Contain(0),
+ "substring needs the query contiguous, so out-of-order words do not match");
+ }
+
+ ///
+ /// Substring diacritic matching is asymmetric. An unmarked query folds
+ /// diacritics (so it matches accented text), but a query that itself contains an accent is treated
+ /// as specific -- it matches only that accent, not the bare letter or a different accent.
+ ///
+ [Test]
+ public void SubstringDiacriticMatch_isAsymmetric()
+ {
+ // Precomposed accented letters, built from code points to keep the source ASCII.
+ string aTilde = ((char)0x00E3).ToString(); // a with tilde
+ string eAcute = ((char)0x00E9).ToString(); // e with acute
+ string eGrave = ((char)0x00E8).ToString(); // e with grave
+
+ var searcher = new StringSearcher(SearchType.Substring, m_wsManager);
+ searcher.Add(0, 0, TsStringUtils.MakeString(aTilde + "pple", m_enWs)); // accented "apple"
+ searcher.Add(1, 0, TsStringUtils.MakeString("apple", m_enWs)); // plain "apple"
+ searcher.Add(2, 0, TsStringUtils.MakeString("caf" + eAcute, m_enWs)); // "cafe" with acute
+ searcher.Add(3, 0, TsStringUtils.MakeString("caf" + eGrave, m_enWs)); // "cafe" with grave
+
+ // Unmarked query folds diacritics: "ap" matches both the accented and the plain word.
+ CheckSearch(searcher, TsStringUtils.MakeString("ap", m_enWs), new[] {0, 1});
+ // A marked query matches its own accented text...
+ CheckSearch(searcher, TsStringUtils.MakeString(aTilde + "p", m_enWs), new[] {0});
+ // ...but not the bare, unaccented text.
+ Assert.That(searcher.Search(0, TsStringUtils.MakeString(aTilde, m_enWs)), Does.Not.Contain(1),
+ "an accented query should not match unaccented text");
+ // A marked query matches only the same accent, not a different one.
+ CheckSearch(searcher, TsStringUtils.MakeString("caf" + eAcute, m_enWs), new[] {2});
+ Assert.That(searcher.Search(0, TsStringUtils.MakeString("caf" + eAcute, m_enWs)), Does.Not.Contain(3),
+ "one accent should not match a different accent");
+ }
+
+ ///
+ /// Substring matching is insensitive to Unicode normalization: a composed character and its
+ /// decomposed (base + combining mark) form match each other, in either direction.
+ ///
+ [Test]
+ public void SubstringMatch_isNormalizationInsensitive()
+ {
+ // Cyrillic short-I: one precomposed code point vs. base + combining breve.
+ string composed = ((char)0x0439).ToString();
+ string decomposed = ((char)0x0438).ToString() + ((char)0x0306).ToString();
+ Assert.That(composed, Is.Not.EqualTo(decomposed), "the two forms should differ byte-for-byte");
+
+ var indexComposed = new StringSearcher(SearchType.Substring, m_wsManager);
+ indexComposed.Add(0, 0, TsStringUtils.MakeString(composed, m_enWs));
+ CheckSearch(indexComposed, TsStringUtils.MakeString(decomposed, m_enWs), new[] {0});
+
+ var indexDecomposed = new StringSearcher(SearchType.Substring, m_wsManager);
+ indexDecomposed.Add(0, 0, TsStringUtils.MakeString(decomposed, m_enWs));
+ CheckSearch(indexDecomposed, TsStringUtils.MakeString(composed, m_enWs), new[] {0});
+ }
}
}