LT-22524: Add substring search mode to StringSearcher

src/SIL.LCModel.Core/Text/StringSearcher.cs

@@ -4,6 +4,7 @@
 
 using System;
 using System.Collections.Generic;
+using System.Globalization;
 using System.Linq;
 using System.Text;
 using Icu;
@@ -30,7 +31,11 @@ public enum SearchType
 		/// <summary>
 		/// Matches any words in a string.
 		/// </summary>
-		FullText
+		FullText,
+		/// <summary>
+		/// Matches any portion within a string.
+		/// </summary>
+		Substring
 	}
 
 	/// <summary>
@@ -119,7 +124,31 @@ public IEnumerable<T> GetItems(byte[] lower, byte[] upper)
 
 		#endregion SortKeyIndex class
 
+		#region SubstringEntry struct
+
+		/// <summary>
+		/// Pairs an indexed item with the raw text scanned for substring matches. Used by
+		/// <see cref="SearchType.Substring"/>.
+		/// </summary>
+		private struct SubstringEntry
+		{
+			private readonly T m_item;
+			private readonly string m_text;
+
+			public SubstringEntry(T item, string text)
+			{
+				m_item = item;
+				m_text = text;
+			}
+
+			public T Item { get { return m_item; } }
+			public string Text { get { return m_text; } }
+		}
+
+		#endregion SubstringEntry struct
+
 		private readonly Dictionary<Tuple<int, int>, SortKeyIndex> m_indices = new Dictionary<Tuple<int, int>, SortKeyIndex>();
+		private readonly Dictionary<Tuple<int, int>, List<SubstringEntry>> m_rawIndices = new Dictionary<Tuple<int, int>, List<SubstringEntry>>();
 		private readonly SearchType m_type;
 		private readonly Func<int, string, byte[]> m_sortKeySelector;
 		private readonly Func<int, string, IEnumerable<string>> m_tokenizer;
@@ -183,18 +212,27 @@ public void Add(T item, int indexId, ITsString tss)
 		/// </summary>
 		public void Add(T item, int indexId, int wsId, string text)
 		{
-			SortKeyIndex index = GetIndex(indexId, wsId);
+			if (string.IsNullOrEmpty(text))
+				return;
+
 			switch (m_type)
 			{
 				case SearchType.Exact:
 				case SearchType.Prefix:
-					index.Add(m_sortKeySelector(wsId, text), item);
+					GetIndex(indexId, wsId).Add(m_sortKeySelector(wsId, text), item);
 					break;
 
 				case SearchType.FullText:
+				{
+					SortKeyIndex index = GetIndex(indexId, wsId);
 					foreach (string token in RemoveWhitespaceAndPunctTokens(m_tokenizer(wsId, text)))
 						index.Add(m_sortKeySelector(wsId, token), item);
 					break;
+				}
+
+				case SearchType.Substring:
+					GetRawIndex(indexId, wsId).Add(new SubstringEntry(item, text));
+					break;
 			}
 		}
 
@@ -233,12 +271,12 @@ public IEnumerable<T> Search(int indexId, int wsId, string text)
 			if (string.IsNullOrEmpty(text))
 				return Enumerable.Empty<T>();
 
-			SortKeyIndex index = GetIndex(indexId, wsId);
 			switch (m_type)
 			{
 				case SearchType.Exact:
 				case SearchType.Prefix:
 					{
+						SortKeyIndex index = GetIndex(indexId, wsId);
 						byte[] sortKey = m_sortKeySelector(wsId, text);
 						var lower = new byte[text.Length * SortKeyFactor];
 						Collator.GetSortKeyBound(sortKey, UColBoundMode.UCOL_BOUND_LOWER, ref lower);
@@ -252,22 +290,40 @@ public IEnumerable<T> Search(int indexId, int wsId, string text)
 					}
 
 				case SearchType.FullText:
-					IEnumerable<T> results = null;
-					string[] tokens = RemoveWhitespaceAndPunctTokens(m_tokenizer(wsId, text)).ToArray();
-					for (int i = 0; i < tokens.Length; i++)
 					{
-						byte[] sortKey = m_sortKeySelector(wsId, tokens[i]);
-						var lower = new byte[tokens[i].Length*SortKeyFactor];
-						Collator.GetSortKeyBound(sortKey, UColBoundMode.UCOL_BOUND_LOWER, ref lower);
-						var upper = new byte[tokens[i].Length*SortKeyFactor];
-						Collator.GetSortKeyBound(sortKey,
-											i < tokens.Length - 1
-												? UColBoundMode.UCOL_BOUND_UPPER
-												: UColBoundMode.UCOL_BOUND_UPPER_LONG, ref upper);
-						IEnumerable<T> items = index.GetItems(lower, upper);
-						results = results == null ? items : results.Intersect(items);
+						SortKeyIndex index = GetIndex(indexId, wsId);
+						IEnumerable<T> results = null;
+						string[] tokens = RemoveWhitespaceAndPunctTokens(m_tokenizer(wsId, text)).ToArray();
+						for (int i = 0; i < tokens.Length; i++)
+						{
+							byte[] sortKey = m_sortKeySelector(wsId, tokens[i]);
+							var lower = new byte[tokens[i].Length*SortKeyFactor];
+							Collator.GetSortKeyBound(sortKey, UColBoundMode.UCOL_BOUND_LOWER, ref lower);
+							var upper = new byte[tokens[i].Length*SortKeyFactor];
+							Collator.GetSortKeyBound(sortKey,
+												i < tokens.Length - 1
+													? UColBoundMode.UCOL_BOUND_UPPER
+													: UColBoundMode.UCOL_BOUND_UPPER_LONG, ref upper);
+							IEnumerable<T> items = index.GetItems(lower, upper);
+							results = results == null ? items : results.Intersect(items);
+						}
+						return results;
+					}
+
+				case SearchType.Substring:
+					{
+						List<SubstringEntry> raw;
+						if (!m_rawIndices.TryGetValue(Tuple.Create(indexId, wsId), out raw))
+							return Enumerable.Empty<T>();
+						CompareInfo ci = CultureInfo.InvariantCulture.CompareInfo;
+						// Fold diacritics only when the search term itself has none: an unmarked query
+						// matches accented text ("cafe" finds "café"), but a query that includes an accent
+						// is treated as specific ("café" does not match a bare "cafe").
+						CompareOptions options = ContainsDiacritic(text)
+							? CompareOptions.IgnoreCase
+							: CompareOptions.IgnoreCase | CompareOptions.IgnoreNonSpace;
+						return raw.Where(entry => ci.IndexOf(entry.Text, text, options) >= 0).Select(entry => entry.Item);
 					}
-					return results;
 			}
 
 			return Enumerable.Empty<T>();
@@ -278,12 +334,22 @@ private static IEnumerable<string> RemoveWhitespaceAndPunctTokens(IEnumerable<st
 			return tokens.Where(t => !t.All(c => Character.IsSpace(c) || Character.IsPunct(c)));
 		}
 
+		/// <summary>
+		/// True if the string contains a diacritic.
+		/// </summary>
+		private static bool ContainsDiacritic(string value)
+		{
+			return value.Normalize(NormalizationForm.FormD)
+				.Any(ch => Character.GetCharType(ch) == Character.UCharCategory.NON_SPACING_MARK);
+		}
+
 		/// <summary>
 		/// Clears all of the indices.
 		/// </summary>
 		public void Clear()
 		{
 			m_indices.Clear();
+			m_rawIndices.Clear();
 		}
 
 		private SortKeyIndex GetIndex(int indexId, int ws)
@@ -299,6 +365,18 @@ private SortKeyIndex GetIndex(int indexId, int ws)
 			return index;
 		}
 
+		private List<SubstringEntry> GetRawIndex(int indexId, int ws)
+		{
+			var key = Tuple.Create(indexId, ws);
+			List<SubstringEntry> list;
+			if (!m_rawIndices.TryGetValue(key, out list))
+			{
+				list = new List<SubstringEntry>();
+				m_rawIndices[key] = list;
+			}
+			return list;
+		}
+
 		private static IEnumerable<Tuple<int, string>> GetWsStrings(ITsString tss)
 		{
 			var sb = new StringBuilder();

tests/SIL.LCModel.Core.Tests/Text/StringSearcherTests.cs

@@ -85,12 +85,13 @@ public void PrefixSearchTest()
 		}
 
 		/// <summary>
-		/// Tests prefix matching.
+		/// Builds the shared multi-writing-system corpus used by both <see cref="FullTextSearchTest"/>
+		/// and <see cref="SubstringResultsIncludeAllFullTextResults"/>. Item 2 deliberately mixes a
+		/// French run and an English run.
 		/// </summary>
-		[Test]
-		public void FullTextSearchTest()
+		private StringSearcher<int> BuildMultiRunCorpus(SearchType type)
 		{
-			var searcher = new StringSearcher<int>(SearchType.FullText, m_wsManager);
+			var searcher = new StringSearcher<int>(type, m_wsManager);
 			searcher.Add(0, 0, TsStringUtils.MakeString("test", m_enWs));
 			searcher.Add(1, 0, TsStringUtils.MakeString("c'est une phrase", m_frWs));
 			ITsIncStrBldr tisb = TsStringUtils.MakeIncStrBldr();
@@ -100,11 +101,172 @@ public void FullTextSearchTest()
 			tisb.Append("We use it for testing purposes.");
 			searcher.Add(2, 0, tisb.GetString());
 			searcher.Add(3, 0, TsStringUtils.MakeString("Hello, how are you doing? I am doing fine. That is good to know.", m_enWs));
+			return searcher;
+		}
+
+		/// <summary>
+		/// The queries exercised by <see cref="FullTextSearchTest"/>, so the substring-superset test
+		/// covers exactly the same scenarios. These are all single tokens or contiguous, in-order
+		/// phrases, and that is deliberate: substring is a superset of full-text ONLY for those shapes
+		/// (full-text ANDs word tokens regardless of order, while substring needs the whole query to
+		/// appear contiguously). Adding an out-of-order multi-word query here would make
+		/// <see cref="SubstringResultsIncludeAllFullTextResults"/> fail; that boundary is demonstrated
+		/// by <see cref="Substring_isNotASupersetForOutOfOrderMultiWordQueries"/>.
+		/// </summary>
+		private ITsString[] FullTextQueries()
+		{
+			return new[]
+			{
+				TsStringUtils.MakeString("test", m_enWs),
+				TsStringUtils.MakeString("c'est une", m_frWs),
+				TsStringUtils.MakeString("t", m_enWs),
+				TsStringUtils.MakeString("testing purpose", m_enWs)
+			};
+		}
+
+		/// <summary>
+		/// Tests full-text (word/prefix) matching.
+		/// </summary>
+		[Test]
+		public void FullTextSearchTest()
+		{
+			var searcher = BuildMultiRunCorpus(SearchType.FullText);
 
 			CheckSearch(searcher, TsStringUtils.MakeString("test", m_enWs), new[] {0, 2});
 			CheckSearch(searcher, TsStringUtils.MakeString("c'est une", m_frWs), new[] {1, 2});
 			CheckSearch(searcher, TsStringUtils.MakeString("t", m_enWs), new[] {0, 2, 3});
 			CheckSearch(searcher, TsStringUtils.MakeString("testing purpose", m_enWs), new[] {2});
 		}
+
+		/// <summary>
+		/// Tests substring (match-anywhere) matching, including infix, case- and diacritic-insensitivity.
+		/// </summary>
+		[Test]
+		public void SubstringSearchTest()
+		{
+			var searcher = new StringSearcher<int>(SearchType.Substring, m_wsManager);
+			searcher.Add(0, 0, TsStringUtils.MakeString("language", m_enWs));
+			searcher.Add(1, 0, TsStringUtils.MakeString("gauge", m_enWs));
+			searcher.Add(2, 0, TsStringUtils.MakeString("résumé", m_frWs));
+			searcher.Add(3, 0, TsStringUtils.MakeString("zebra", m_enWs));
+
+			// infix match: "uage" is not a prefix of "language" but is a substring (fails under Prefix/FullText).
+			CheckSearch(searcher, TsStringUtils.MakeString("uage", m_enWs), new[] {0});
+			// interior substring
+			CheckSearch(searcher, TsStringUtils.MakeString("gua", m_enWs), new[] {0});
+			CheckSearch(searcher, TsStringUtils.MakeString("aug", m_enWs), new[] {1});
+			// case-insensitive
+			CheckSearch(searcher, TsStringUtils.MakeString("LANG", m_enWs), new[] {0});
+			// diacritic-insensitive
+			CheckSearch(searcher, TsStringUtils.MakeString("resume", m_frWs), new[] {2});
+			// whole-string still matches
+			CheckSearch(searcher, TsStringUtils.MakeString("zebra", m_enWs), new[] {3});
+			// no match anywhere
+			CheckNoResultsSearch(searcher, TsStringUtils.MakeString("xyz", m_enWs));
+		}
+
+		/// <summary>
+		/// Substring search must not miss anything a full-text search would find on the same corpus and
+		/// queries: its result set is a near superset 
+		/// (see <see cref="Substring_isNotASupersetForOutOfOrderMultiWordQueries"/>) 
+		/// of the full-text result set. This guards the promise that switching Find Lexical Entry to 
+		/// substring never drops a result that used to appear.
+		/// (This is a superset, not equality: substring also returns extra infix matches.)
+		/// </summary>
+		[Test]
+		public void SubstringResultsIncludeAllFullTextResults()
+		{
+			var fullText = BuildMultiRunCorpus(SearchType.FullText);
+			var substring = BuildMultiRunCorpus(SearchType.Substring);
+
+			foreach (ITsString query in FullTextQueries())
+			{
+				// StringSearcher.Search can return the same item several times (once per matching word);
+				// the real consumer (SearchEngine) dedupes via a HashSet, so compare as sets here too.
+				int[] fullTextResults = fullText.Search(0, query).Distinct().ToArray();
+				Assert.That(fullTextResults, Is.Not.Empty,
+					"query '" + query.Text + "' should match something under full-text (otherwise the check is vacuous)");
+				Assert.That(substring.Search(0, query).Distinct(), Is.SupersetOf(fullTextResults),
+					"substring dropped a full-text match for query '" + query.Text + "'");
+			}
+		}
+
+		/// <summary>
+		/// Pins the boundary of the superset guarantee: it holds only for single-token or contiguous,
+		/// in-order queries. A multi-word query whose words appear OUT OF ORDER matches under full-text
+		/// (which ANDs the word tokens regardless of order) but NOT under substring (which needs the
+		/// whole query to appear contiguously). This is the concrete case behind the scoping note on
+		/// <see cref="FullTextQueries"/>.
+		/// </summary>
+		[Test]
+		public void Substring_isNotASupersetForOutOfOrderMultiWordQueries()
+		{
+			var fullText = new StringSearcher<int>(SearchType.FullText, m_wsManager);
+			var substring = new StringSearcher<int>(SearchType.Substring, m_wsManager);
+			ITsString text = TsStringUtils.MakeString("alpha beta gamma", m_enWs);
+			fullText.Add(0, 0, text);
+			substring.Add(0, 0, text);
+
+			// Words present but in a different order than the text.
+			ITsString outOfOrder = TsStringUtils.MakeString("gamma alpha", m_enWs);
+
+			Assert.That(fullText.Search(0, outOfOrder), Does.Contain(0),
+				"full-text ANDs the word tokens, so it matches the words in any order");
+			Assert.That(substring.Search(0, outOfOrder), Does.Not.Contain(0),
+				"substring needs the query contiguous, so out-of-order words do not match");
+		}
+
+		/// <summary>
+		/// Substring diacritic matching is asymmetric. An unmarked query folds
+		/// diacritics (so it matches accented text), but a query that itself contains an accent is treated
+		/// as specific -- it matches only that accent, not the bare letter or a different accent.
+		/// </summary>
+		[Test]
+		public void SubstringDiacriticMatch_isAsymmetric()
+		{
+			// Precomposed accented letters, built from code points to keep the source ASCII.
+			string aTilde = ((char)0x00E3).ToString(); // a with tilde
+			string eAcute = ((char)0x00E9).ToString(); // e with acute
+			string eGrave = ((char)0x00E8).ToString(); // e with grave
+
+			var searcher = new StringSearcher<int>(SearchType.Substring, m_wsManager);
+			searcher.Add(0, 0, TsStringUtils.MakeString(aTilde + "pple", m_enWs)); // accented "apple"
+			searcher.Add(1, 0, TsStringUtils.MakeString("apple", m_enWs));         // plain "apple"
+			searcher.Add(2, 0, TsStringUtils.MakeString("caf" + eAcute, m_enWs));  // "cafe" with acute
+			searcher.Add(3, 0, TsStringUtils.MakeString("caf" + eGrave, m_enWs));  // "cafe" with grave
+
+			// Unmarked query folds diacritics: "ap" matches both the accented and the plain word.
+			CheckSearch(searcher, TsStringUtils.MakeString("ap", m_enWs), new[] {0, 1});
+			// A marked query matches its own accented text...
+			CheckSearch(searcher, TsStringUtils.MakeString(aTilde + "p", m_enWs), new[] {0});
+			// ...but not the bare, unaccented text.
+			Assert.That(searcher.Search(0, TsStringUtils.MakeString(aTilde, m_enWs)), Does.Not.Contain(1),
+				"an accented query should not match unaccented text");
+			// A marked query matches only the same accent, not a different one.
+			CheckSearch(searcher, TsStringUtils.MakeString("caf" + eAcute, m_enWs), new[] {2});
+			Assert.That(searcher.Search(0, TsStringUtils.MakeString("caf" + eAcute, m_enWs)), Does.Not.Contain(3),
+				"one accent should not match a different accent");
+		}
+
+		/// <summary>
+		/// Substring matching is insensitive to Unicode normalization: a composed character and its
+		/// decomposed (base + combining mark) form match each other, in either direction.
+		/// </summary>
+		[Test]
+		public void SubstringMatch_isNormalizationInsensitive()
+		{
+			// Cyrillic short-I: one precomposed code point vs. base + combining breve.
+			string composed = ((char)0x0439).ToString();
+			string decomposed = ((char)0x0438).ToString() + ((char)0x0306).ToString();
+			Assert.That(composed, Is.Not.EqualTo(decomposed), "the two forms should differ byte-for-byte");
+
+			var indexComposed = new StringSearcher<int>(SearchType.Substring, m_wsManager);
+			indexComposed.Add(0, 0, TsStringUtils.MakeString(composed, m_enWs));
+			CheckSearch(indexComposed, TsStringUtils.MakeString(decomposed, m_enWs), new[] {0});
+
+			var indexDecomposed = new StringSearcher<int>(SearchType.Substring, m_wsManager);
+			indexDecomposed.Add(0, 0, TsStringUtils.MakeString(decomposed, m_enWs));
+			CheckSearch(indexDecomposed, TsStringUtils.MakeString(composed, m_enWs), new[] {0});
+		}
 	}
 }