pucene · wachterjohannes · Jun 10, 2017
diff --git a/src/Component/Client/IndexInterface.php b/src/Component/Client/IndexInterface.php
@@ -36,4 +36,6 @@ public function search(Search $search, $type);
      * @return array
      */
     public function get($type, $id);
+
+    public function optimize();
 }
diff --git a/src/Component/Elasticsearch/ElasticsearchIndex.php b/src/Component/Elasticsearch/ElasticsearchIndex.php
@@ -105,4 +105,12 @@ public function get($type, $id)
 
         return $response;
     }
+
+    /**
+     * {@inheritdoc}
+     */
+    public function optimize()
+    {
+        // nothing to optimize currently.
+    }
 }
diff --git a/src/Component/Pucene/Dbal/DbalStorage.php b/src/Component/Pucene/Dbal/DbalStorage.php
@@ -144,6 +144,11 @@ public function termStatistics()
         return new DbalTermStatistics($this->connection, $this->getSchema());
     }
 
+    public function optimize()
+    {
+        $this->persister->optimize();
+    }
+
     public function getConnection()
     {
         return $this->connection;

diff --git a/src/Component/Pucene/Dbal/DocumentPersister.php b/src/Component/Pucene/Dbal/DocumentPersister.php
@@ -56,21 +56,51 @@ public function persist(Document $document, array $fields)
                     $token->getEncodedTerm(),
                     ElasticsearchPrecision::fieldNorm($field->getNumberOfTerms())
                 );
+
+                $this->connection->createQueryBuilder()
+                    ->update($this->schema->getDocumentTermsTableName())
+                    ->set('frequency', 'frequency + 1')
+                    ->andWhere('field_name = :fieldName')
+                    ->andWhere('term = :term')
+                    ->setParameter('fieldName', $field->getName())
+                    ->setParameter('term', $token->getEncodedTerm())
+                    ->execute();
             }
 
             // update term frequency
             foreach ($fieldTerms as $term => $frequency) {
-                $this->connection->update(
-                    $this->schema->getDocumentTermsTableName(),
-                    [
-                        'term_frequency' => $frequency,
-                    ],
-                    ['document_id' => $document->getId(), 'field_name' => $field->getName(), 'term' => $term]
-                );
+                $this->connection->createQueryBuilder()
+                    ->update($this->schema->getDocumentTermsTableName())
+                    ->set('term_frequency', sqrt($frequency))
+                    ->set('score', 'field_norm * ' . sqrt($frequency))
+                    ->andWhere('document_ID = :document')
+                    ->andWhere('field_name = :fieldName')
+                    ->andWhere('term = :term')
+                    ->setParameter('document', $document->getId())
+                    ->setParameter('fieldName', $field->getName())
+                    ->setParameter('term', $term)
+                    ->execute();
             }
         }
     }
 
+    public function optimize()
+    {
+        // TODO recalculate term frequency
+
+        $docCount = $this->connection->createQueryBuilder()
+            ->select('COUNT(id)')
+            ->from($this->schema->getDocumentsTableName())
+            ->execute()
+            ->fetchColumn();
+
+        // calculate inverse-document-frequency
+        $this->connection->createQueryBuilder()
+            ->update($this->schema->getDocumentTermsTableName())
+            ->set('idf', '1 + log(' . $docCount . ' / (frequency + 1))')
+            ->execute();
+    }
+
     /**
      * @param Document $document
      */
@@ -101,13 +131,24 @@ protected function insertDocument(Document $document)
      */
     protected function insertToken(string $documentId, string $fieldName, $term, $fieldNorm)
     {
+        $frequency = $this->connection->createQueryBuilder()
+            ->select('frequency')
+            ->from($this->schema->getDocumentTermsTableName())
+            ->andWhere('field_name = :fieldName')
+            ->andWhere('term = :term')
+            ->setParameter('fieldName', $fieldName)
+            ->setParameter('term', $term)
+            ->execute()
+            ->fetchColumn();
+
         $this->connection->insert(
             $this->schema->getDocumentTermsTableName(),
             [
                 'document_id' => $documentId,
                 'field_name' => $fieldName,
                 'term' => $term,
                 'field_norm' => $fieldNorm,
+                'frequency' => $frequency ?: 0,
             ]
         );
     }

diff --git a/src/Component/Pucene/Dbal/Interpreter/Element/TermInterpreter.php b/src/Component/Pucene/Dbal/Interpreter/Element/TermInterpreter.php
@@ -19,9 +19,7 @@ public function interpret(ElementInterface $element, PuceneQueryBuilder $queryBu
     {
         $expr = $queryBuilder->expr();
 
-        return $expr->isNotNull(
-            $queryBuilder->joinTerm($element->getField(), $element->getTerm()) . '.id'
-        );
+        return $expr->isNotNull($queryBuilder->joinTerm($element->getField(), $element->getTerm()) . '.id');
     }
 
     /**

diff --git a/src/Component/Pucene/Dbal/Math/TermFrequency.php b/src/Component/Pucene/Dbal/Math/TermFrequency.php
@@ -50,7 +50,7 @@ public function __construct(
     public function __toString(): string
     {
         return $this->expr->coalesce(
-            $this->expr->sqrt($this->expr->variable($this->queryBuilder->joinTerm($this->field, $this->term) . '.term_frequency')),
+            $this->expr->variable($this->queryBuilder->joinTerm($this->field, $this->term) . '.term_frequency'),
             $this->expr->value(0)
         );
     }

diff --git a/src/Component/Pucene/Dbal/PuceneSchema.php b/src/Component/Pucene/Dbal/PuceneSchema.php
@@ -60,6 +60,9 @@ private function createDocumentTermsTable()
         $fields->addColumn('term', 'string', ['length' => 255]);
         $fields->addColumn('term_frequency', 'integer', ['default' => 0]);
         $fields->addColumn('field_norm', 'float', ['default' => 0]);
+        $fields->addColumn('score', 'float', ['default' => 0]);
+        $fields->addColumn('frequency', 'integer', ['default' => 0]);
+        $fields->addColumn('idf', 'float', ['default' => 0]);
 
         $fields->setPrimaryKey(['id']);
         $fields->addForeignKeyConstraint(

diff --git a/src/Component/Pucene/Dbal/ScoringAlgorithm.php b/src/Component/Pucene/Dbal/ScoringAlgorithm.php
@@ -7,8 +7,6 @@
 use Pucene\Component\Pucene\Compiler\Element\TermElement;
 use Pucene\Component\Pucene\Compiler\ElementInterface;
 use Pucene\Component\Pucene\Dbal\Interpreter\PuceneQueryBuilder;
-use Pucene\Component\Pucene\Dbal\Math\FieldLengthNorm;
-use Pucene\Component\Pucene\Dbal\Math\TermFrequency;
 use Pucene\Component\Symfony\Pool\PoolInterface;
 
 class ScoringAlgorithm
@@ -38,6 +36,11 @@ class ScoringAlgorithm
      */
     private $docCount;
 
+    /**
+     * @var int[]
+     */
+    private $docCounts = [];
+
     /**
      * @param PuceneQueryBuilder $queryBuilder
      * @param PuceneSchema $schema
@@ -54,17 +57,21 @@ public function __construct(PuceneQueryBuilder $queryBuilder, PuceneSchema $sche
 
     public function scoreTerm(TermElement $element, float $queryNorm = null, float $boost = 1)
     {
-        $idf = $this->inverseDocumentFrequency($element);
+        $termName = $this->queryBuilder->joinTerm($element->getField(), $element->getTerm());
+        $expression = $this->math->multiply();
+        $expression->add($this->math->coalesce($this->math->variable($termName . '.idf'), $this->math->value(0)));
+
+        if ($boost != 1) {
+            $expression->add($this->math->value($boost));
+        }
 
-        $factor = $idf * $boost;
         if ($queryNorm) {
-            $factor *= $idf * $queryNorm;
+            $expression->add($this->math->coalesce($this->math->variable($termName . '.idf'), $this->math->value(0)));
+            $expression->add($this->math->value($queryNorm));
         }
 
-        return $this->math->multiply(
-            new TermFrequency($element->getField(), $element->getTerm(), $this->queryBuilder),
-            new FieldLengthNorm($element->getField(), $element->getTerm(), $this->queryBuilder, $this->math),
-            $this->math->value($factor)
+        return $expression->add(
+            $this->math->coalesce($this->math->variable($termName . '.score'), $this->math->value(0))
         );
     }
 
@@ -108,13 +115,18 @@ private function inverseDocumentFrequency(ElementInterface $element): float
      *
      * @return float
      */
-    private function calculateInverseDocumentFrequency($docCount)
+    public function calculateInverseDocumentFrequency($docCount)
     {
         return 1 + log((float) $this->getDocCount() / ($docCount + 1));
     }
 
     private function getDocCountForElement(ElementInterface $element)
     {
+        $key = $element->getField() . $element->getTerm();
+        if (array_key_exists($key, $this->docCounts)) {
+            return $this->docCounts[$key];
+        }
+
         $queryBuilder = (new PuceneQueryBuilder($this->queryBuilder->getConnection(), $this->schema))
             ->select('count(document.id) as count')
             ->from($this->schema->getDocumentsTableName(), 'document');
@@ -124,10 +136,10 @@ private function getDocCountForElement(ElementInterface $element)
             $queryBuilder->where($expression);
         }
 
-        return (int) $queryBuilder->execute()->fetchColumn();
+        return $this->docCounts[$key] = (int) $queryBuilder->execute()->fetchColumn();
     }
 
-    private function getDocCount()
+    public function getDocCount()
     {
         if ($this->docCount) {
             return $this->docCount;

diff --git a/src/Component/Pucene/PuceneIndex.php b/src/Component/Pucene/PuceneIndex.php
@@ -96,4 +96,9 @@ public function get($type, $id)
     {
         return $this->storage->get($type, $id);
     }
+
+    public function optimize()
+    {
+        $this->storage->optimize();
+    }
 }
diff --git a/src/Component/Pucene/StorageInterface.php b/src/Component/Pucene/StorageInterface.php
@@ -20,4 +20,6 @@ public function search(Search $search, $type);
     public function get($type, $id);
 
     public function termStatistics();
+
+    public function optimize();
 }
diff --git a/tests/src/TestBundle/Command/ImportJsonCommand.php b/tests/src/TestBundle/Command/ImportJsonCommand.php
@@ -2,6 +2,7 @@
 
 namespace Pucene\Tests\TestBundle\Command;
 
+use Pucene\Component\Client\ClientInterface;
 use Symfony\Bundle\FrameworkBundle\Command\ContainerAwareCommand;
 use Symfony\Component\Console\Helper\ProgressBar;
 use Symfony\Component\Console\Input\InputArgument;
@@ -30,6 +31,7 @@ protected function configure()
      */
     protected function execute(InputInterface $input, OutputInterface $output)
     {
+        /** @var ClientInterface $client */
         $client = $this->getContainer()->get('pucene.' . $input->getOption('adapter') . '.client');
         $index = $client->get($input->getArgument('index'));
 
@@ -44,6 +46,8 @@ protected function execute(InputInterface $input, OutputInterface $output)
             $progressBar->advance();
         }
 
+        $index->optimize();
+
         $progressBar->finish();
     }
 }
diff --git a/tests/src/TestBundle/Command/ImportWikidataCommand.php b/tests/src/TestBundle/Command/ImportWikidataCommand.php
@@ -58,6 +58,8 @@ protected function execute(InputInterface $input, OutputInterface $output)
             $progressBar->advance();
         }
 
+        $index->optimize();
+
         $progressBar->finish();
     }
 }