Remove dynamic dispatch, reformulate TokenStream as Iterators

examples/pre_tokenized_text.rs

@@ -17,12 +17,7 @@ use tantivy::{doc, Index, ReloadPolicy};
 use tempfile::TempDir;
 
 fn pre_tokenize_text(text: &str) -> Vec<Token> {
-    let mut token_stream = SimpleTokenizer.token_stream(text);
-    let mut tokens = vec![];
-    while token_stream.advance() {
-        tokens.push(token_stream.token().clone());
-    }
-    tokens
+    SimpleTokenizer.token_stream(text).collect()
 }
 
 fn main() -> tantivy::Result<()> {

examples/snippet.rs

@@ -51,7 +51,7 @@ fn main() -> tantivy::Result<()> {
 
     let top_docs = searcher.search(&query, &TopDocs::with_limit(10))?;
 
-    let snippet_generator = SnippetGenerator::create(&searcher, &*query, body)?;
+    let mut snippet_generator = SnippetGenerator::create(&searcher, &*query, body)?;
 
     for (score, doc_address) in top_docs {
         let doc = searcher.doc(doc_address)?;

examples/stop_words.rs

@@ -50,12 +50,13 @@ fn main() -> tantivy::Result<()> {
 
     // This tokenizer lowers all of the text (to help with stop word matching)
     // then removes all instances of `the` and `and` from the corpus
-    let tokenizer = TextAnalyzer::from(SimpleTokenizer)
-        .filter(LowerCaser)
+    let tokenizer = analyzer_builder(SimpleTokenizer)
+        .filter(LowerCaser::new())
         .filter(StopWordFilter::remove(vec![
             "the".to_string(),
             "and".to_string(),
-        ]));
+        ]))
+        .build();
 
     index.tokenizers().register("stoppy", tokenizer);
 

src/core/index.rs

@@ -20,7 +20,7 @@ use crate::reader::IndexReaderBuilder;
 use crate::schema::Field;
 use crate::schema::FieldType;
 use crate::schema::Schema;
-use crate::tokenizer::{TextAnalyzer, TokenizerManager};
+use crate::tokenizer::{TextAnalyzerT, TokenizerManager};
 use crate::IndexWriter;
 use std::collections::HashSet;
 use std::fmt;
@@ -119,13 +119,12 @@ impl Index {
             return Index::create(dir, schema);
         }
         let index = Index::open(dir)?;
-        if index.schema() == schema {
-            Ok(index)
-        } else {
-            Err(TantivyError::SchemaError(
+        if index.schema() != schema {
+            return Err(TantivyError::SchemaError(
                 "An index exists but the schema does not match.".to_string(),
-            ))
+            ));
         }
+        Ok(index)
     }
 
     /// Creates a new index in a temp directory.
@@ -181,11 +180,11 @@ impl Index {
     }
 
     /// Helper to access the tokenizer associated to a specific field.
-    pub fn tokenizer_for_field(&self, field: Field) -> crate::Result<TextAnalyzer> {
+    pub fn tokenizer_for_field(&self, field: Field) -> crate::Result<Box<dyn TextAnalyzerT>> {
         let field_entry = self.schema.get_field_entry(field);
         let field_type = field_entry.field_type();
         let tokenizer_manager: &TokenizerManager = self.tokenizers();
-        let tokenizer_name_opt: Option<TextAnalyzer> = match field_type {
+        let tokenizer_name_opt: Option<Box<dyn TextAnalyzerT>> = match field_type {
             FieldType::Str(text_options) => text_options
                 .get_indexing_options()
                 .map(|text_indexing_options| text_indexing_options.tokenizer().to_string())

src/core/segment_reader.rs

@@ -310,7 +310,7 @@ impl SegmentReader {
     }
 
     /// Returns an iterator that will iterate over the alive document ids
-    pub fn doc_ids_alive<'a>(&'a self) -> impl Iterator<Item = DocId> + 'a {
+    pub fn doc_ids_alive(&self) -> impl Iterator<Item = DocId> + '_ {
         (0u32..self.max_doc).filter(move |doc| !self.is_deleted(*doc))
     }
 

src/indexer/segment_writer.rs

@@ -10,10 +10,9 @@ use crate::schema::FieldType;
 use crate::schema::Schema;
 use crate::schema::Term;
 use crate::schema::Value;
-use crate::schema::{Field, FieldEntry};
-use crate::tokenizer::{BoxTokenStream, PreTokenizedStream};
-use crate::tokenizer::{FacetTokenizer, TextAnalyzer};
-use crate::tokenizer::{TokenStreamChain, Tokenizer};
+use crate::tokenizer::PreTokenizedStream;
+use crate::tokenizer::{DynTokenStreamChain, Tokenizer};
+use crate::tokenizer::{FacetTokenizer, TextAnalyzerT, Token};
 use crate::Opstamp;
 use crate::{DocId, SegmentComponent};
 
@@ -23,7 +22,7 @@ use crate::{DocId, SegmentComponent};
 fn initial_table_size(per_thread_memory_budget: usize) -> crate::Result<usize> {
     let table_memory_upper_bound = per_thread_memory_budget / 3;
     if let Some(limit) = (10..)
-        .take_while(|num_bits: &usize| compute_table_size(*num_bits) < table_memory_upper_bound)
+        .take_while(|&num_bits| compute_table_size(num_bits) < table_memory_upper_bound)
         .last()
     {
         Ok(limit.min(19)) // we cap it at 2^19 = 512K.
@@ -45,7 +44,8 @@ pub struct SegmentWriter {
     fast_field_writers: FastFieldsWriter,
     fieldnorms_writer: FieldNormsWriter,
     doc_opstamps: Vec<Opstamp>,
-    tokenizers: Vec<Option<TextAnalyzer>>,
+    // TODO: change type
+    tokenizers: Vec<Option<Box<dyn TextAnalyzerT>>>,
     term_buffer: Term,
 }
 
@@ -70,17 +70,17 @@ impl SegmentWriter {
         let multifield_postings = MultiFieldPostingsWriter::new(schema, table_num_bits);
         let tokenizers = schema
             .fields()
-            .map(
-                |(_, field_entry): (Field, &FieldEntry)| match field_entry.field_type() {
-                    FieldType::Str(ref text_options) => text_options
+            .map(|(_, field_entry)| match field_entry.field_type() {
+                FieldType::Str(text_options) => {
+                    text_options
                         .get_indexing_options()
                         .and_then(|text_index_option| {
                             let tokenizer_name = &text_index_option.tokenizer();
                             tokenizer_manager.get(tokenizer_name)
-                        }),
-                    _ => None,
-                },
-            )
+                        })
+                }
+                _ => None,
+            })
             .collect();
         Ok(SegmentWriter {
             max_doc: 0,
@@ -141,13 +141,13 @@ impl SegmentWriter {
             }
             let (term_buffer, multifield_postings) =
                 (&mut self.term_buffer, &mut self.multifield_postings);
-            match *field_entry.field_type() {
+            match field_entry.field_type() {
                 FieldType::HierarchicalFacet => {
                     term_buffer.set_field(field);
                     let facets =
                         field_values
                             .iter()
-                            .flat_map(|field_value| match *field_value.value() {
+                            .flat_map(|field_value| match field_value.value() {
                                 Value::Facet(ref facet) => Some(facet.encoded_str()),
                                 _ => {
                                     panic!("Expected hierarchical facet");
@@ -157,12 +157,13 @@ impl SegmentWriter {
                         let mut unordered_term_id_opt = None;
                         FacetTokenizer
                             .token_stream(facet_str)
-                            .process(&mut |token| {
+                            .map(|token| {
                                 term_buffer.set_text(&token.text);
                                 let unordered_term_id =
                                     multifield_postings.subscribe(doc_id, &term_buffer);
                                 unordered_term_id_opt = Some(unordered_term_id);
-                            });
+                            })
+                            .count();
                         if let Some(unordered_term_id) = unordered_term_id_opt {
                             self.fast_field_writers
                                 .get_multivalue_writer(field)
@@ -172,37 +173,38 @@ impl SegmentWriter {
                     }
                 }
                 FieldType::Str(_) => {
-                    let mut token_streams: Vec<BoxTokenStream> = vec![];
-                    let mut offsets = vec![];
+                    let mut streams_with_offsets = vec![];
                     let mut total_offset = 0;
 
                     for field_value in field_values {
                         match field_value.value() {
                             Value::PreTokStr(tok_str) => {
-                                offsets.push(total_offset);
+                                streams_with_offsets.push((
+                                    Box::new(PreTokenizedStream::from(tok_str.clone()))
+                                        as Box<dyn Iterator<Item = Token>>,
+                                    total_offset,
+                                ));
                                 if let Some(last_token) = tok_str.tokens.last() {
                                     total_offset += last_token.offset_to;
                                 }
-                                token_streams
-                                    .push(PreTokenizedStream::from(tok_str.clone()).into());
                             }
-                            Value::Str(ref text) => {
+                            Value::Str(text) => {
                                 if let Some(ref mut tokenizer) =
                                     self.tokenizers[field.field_id() as usize]
                                 {
-                                    offsets.push(total_offset);
+                                    streams_with_offsets
+                                        .push((tokenizer.token_stream(text), total_offset));
                                     total_offset += text.len();
-                                    token_streams.push(tokenizer.token_stream(text));
                                 }
                             }
                             _ => (),
                         }
                     }
 
-                    let num_tokens = if token_streams.is_empty() {
+                    let num_tokens = if streams_with_offsets.is_empty() {
                         0
                     } else {
-                        let mut token_stream = TokenStreamChain::new(offsets, token_streams);
+                        let mut token_stream = DynTokenStreamChain::from_vec(streams_with_offsets);
                         multifield_postings.index_text(
                             doc_id,
                             field,
@@ -213,71 +215,62 @@ impl SegmentWriter {
 
                     self.fieldnorms_writer.record(doc_id, field, num_tokens);
                 }
-                FieldType::U64(ref int_option) => {
-                    if int_option.is_indexed() {
-                        for field_value in field_values {
-                            term_buffer.set_field(field_value.field());
-                            let u64_val = field_value
-                                .value()
-                                .u64_value()
-                                .ok_or_else(make_schema_error)?;
-                            term_buffer.set_u64(u64_val);
-                            multifield_postings.subscribe(doc_id, &term_buffer);
-                        }
+                FieldType::U64(int_option) if int_option.is_indexed() => {
+                    for field_value in field_values {
+                        term_buffer.set_field(field_value.field());
+                        let u64_val = field_value
+                            .value()
+                            .u64_value()
+                            .ok_or_else(make_schema_error)?;
+                        term_buffer.set_u64(u64_val);
+                        multifield_postings.subscribe(doc_id, &term_buffer);
                     }
                 }
-                FieldType::Date(ref int_option) => {
-                    if int_option.is_indexed() {
-                        for field_value in field_values {
-                            term_buffer.set_field(field_value.field());
-                            let date_val = field_value
-                                .value()
-                                .date_value()
-                                .ok_or_else(make_schema_error)?;
-                            term_buffer.set_i64(date_val.timestamp());
-                            multifield_postings.subscribe(doc_id, &term_buffer);
-                        }
+                FieldType::Date(int_option) if int_option.is_indexed() => {
+                    for field_value in field_values {
+                        term_buffer.set_field(field_value.field());
+                        let date_val = field_value
+                            .value()
+                            .date_value()
+                            .ok_or_else(make_schema_error)?;
+                        term_buffer.set_i64(date_val.timestamp());
+                        multifield_postings.subscribe(doc_id, &term_buffer);
                     }
                 }
-                FieldType::I64(ref int_option) => {
-                    if int_option.is_indexed() {
-                        for field_value in field_values {
-                            term_buffer.set_field(field_value.field());
-                            let i64_val = field_value
-                                .value()
-                                .i64_value()
-                                .ok_or_else(make_schema_error)?;
-                            term_buffer.set_i64(i64_val);
-                            multifield_postings.subscribe(doc_id, &term_buffer);
-                        }
+                FieldType::I64(int_option) if int_option.is_indexed() => {
+                    for field_value in field_values {
+                        term_buffer.set_field(field_value.field());
+                        let i64_val = field_value
+                            .value()
+                            .i64_value()
+                            .ok_or_else(make_schema_error)?;
+                        term_buffer.set_i64(i64_val);
+                        multifield_postings.subscribe(doc_id, &term_buffer);
                     }
                 }
-                FieldType::F64(ref int_option) => {
-                    if int_option.is_indexed() {
-                        for field_value in field_values {
-                            term_buffer.set_field(field_value.field());
-                            let f64_val = field_value
-                                .value()
-                                .f64_value()
-                                .ok_or_else(make_schema_error)?;
-                            term_buffer.set_f64(f64_val);
-                            multifield_postings.subscribe(doc_id, &term_buffer);
-                        }
+                FieldType::F64(int_option) if int_option.is_indexed() => {
+                    for field_value in field_values {
+                        term_buffer.set_field(field_value.field());
+                        let f64_val = field_value
+                            .value()
+                            .f64_value()
+                            .ok_or_else(make_schema_error)?;
+                        term_buffer.set_f64(f64_val);
+                        multifield_postings.subscribe(doc_id, &term_buffer);
                     }
                 }
-                FieldType::Bytes(ref option) => {
-                    if option.is_indexed() {
-                        for field_value in field_values {
-                            term_buffer.set_field(field_value.field());
-                            let bytes = field_value
-                                .value()
-                                .bytes_value()
-                                .ok_or_else(make_schema_error)?;
-                            term_buffer.set_bytes(bytes);
-                            self.multifield_postings.subscribe(doc_id, &term_buffer);
-                        }
+                FieldType::Bytes(option) if option.is_indexed() => {
+                    for field_value in field_values {
+                        term_buffer.set_field(field_value.field());
+                        let bytes = field_value
+                            .value()
+                            .bytes_value()
+                            .ok_or_else(make_schema_error)?;
+                        term_buffer.set_bytes(bytes);
+                        self.multifield_postings.subscribe(doc_id, &term_buffer);
                     }
                 }
+                _ => {}
             }
         }
         doc.filter_fields(|field| schema.get_field_entry(field).is_stored());

src/positions/reader.rs

@@ -132,7 +132,7 @@ impl PositionReader {
             "offset arguments should be increasing."
         );
         let delta_to_block_offset = offset as i64 - self.block_offset as i64;
-        if delta_to_block_offset < 0 || delta_to_block_offset >= 128 {
+        if !(0..128).contains(&delta_to_block_offset) {
             // The first position is not within the first block.
             // We need to decompress the first block.
             let delta_to_anchor_offset = offset - self.anchor_offset;

src/postings/block_search.rs

@@ -109,9 +109,9 @@ impl BlockSearcher {
     /// The results should be equivalent to
     /// ```compile_fail
     /// block[..]
-    //       .iter()
-    //       .take_while(|&&val| val < target)
-    //       .count()
+    ///       .iter()
+    ///       .take_while(|&&val| val < target)
+    ///       .count()
     /// ```
     ///
     /// The `start` argument is just used to hint that the response is