Added 'bufferedadd' plugin including an example and unittests

examples/7.5-plugin-bufferedadd.php

@@ -0,0 +1,51 @@
+<?php
+require('init.php');
+
+// this very simple plugin is used to show some events
+class simpleDebug extends Solarium_Plugin_Abstract
+{
+    protected $_output = array();
+
+    public function display()
+    {
+        echo implode('<br/>', $this->_output);
+    }
+
+    public function eventBufferedAddFlushStart($buffer) {
+        $this->_output[] = 'Flushing buffer (' . count($buffer) . 'docs)';
+    }
+}
+
+htmlHeader();
+
+// create a client instance and autoload the buffered add plugin
+$client = new Solarium_Client($config);
+$buffer = $client->getPlugin('bufferedadd');
+$buffer->setBufferSize(10); // this is quite low, in most cases you can use a much higher value
+
+// also register a plugin for outputting events
+$debug = new simpleDebug();
+$client->registerPlugin('debugger', $debug);
+
+// let's insert 25 docs
+for ($i=1; $i<=25; $i++) {
+
+    // create a new document with dummy data and add it to the buffer
+    $data = array(
+        'id' => 'test_'.$i,
+        'name' => 'test for buffered add',
+        'price' => $i,
+    );
+    $buffer->createDocument($data);
+
+    // alternatively you could create document instances yourself and use the addDocument(s) method
+}
+
+// At this point two flushes will already have been done by the buffer automatically (at the 10th and 20th doc), now
+// manually flush the remainder. Alternatively you can use the commit method if you want to include a commit command.
+$buffer->flush();
+
+// In total 3 flushes (requests) have been sent to Solr. This should be visible in this output:
+$debug->display();
+
+htmlFooter();

examples/index.html

@@ -124,6 +124,7 @@ <h1>Solarium examples</h1>
                 <li><a href="7.2-plugin-postbigrequest.php">7.2 Post Big Requests</a></li>
                 <li><a href="7.3-plugin-customizerequest.php">7.3 Customize Requests</a></li>
                 <li><a href="7.4-plugin-parallelexecution.php">7.4 Parallel Execution</a></li>
+                <li><a href="7.5-plugin-bufferedadd.php">7.5 Buffered Add for documents</a></li>
             </ul>
 
         </ul>

library/Solarium/Client.php

@@ -164,6 +164,7 @@ class Solarium_Client extends Solarium_Configurable
         'postbigrequest' => 'Solarium_Plugin_PostBigRequest',
         'customizerequest' => 'Solarium_Plugin_CustomizeRequest',
         'parallelexecution' => 'Solarium_Plugin_ParallelExecution',
+        'bufferedadd' => 'Solarium_Plugin_BufferedAdd',
     );
 
     /**

library/Solarium/Plugin/Abstract.php

@@ -70,7 +70,7 @@ public function init($client, $options)
     /**
      * Plugin init function
      *
-     * This is an extension point for plugin implemenations.
+     * This is an extension point for plugin implementations.
      * Will be called as soon as $this->_client and options have been set.
      *
      * @return void
@@ -157,15 +157,15 @@ public function preExecute($query)
 
     /**
      * postExecute hook
-     * 
+     *
      * @param Solarium_Query $query
      * @param Solarium_Result $result
      * @return void
      */
     public function postExecute($query, $result)
     {
     }
-    
+
     /**
      * preCreateQuery hook
      *
@@ -179,7 +179,7 @@ public function preCreateQuery($type, $options)
 
     /**
      * postCreateQuery hook
-     * 
+     *
      * @param string $type
      * @param mixed $options
      * @param Solarium_Query

library/Solarium/Plugin/BufferedAdd.php

@@ -0,0 +1,227 @@
+<?php
+/**
+ * Copyright 2011 Bas de Nooijer. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this listof conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ * The views and conclusions contained in the software and documentation are
+ * those of the authors and should not be interpreted as representing official
+ * policies, either expressed or implied, of the copyright holder.
+ *
+ * @copyright Copyright 2011 Bas de Nooijer <[email protected]>
+ * @license http://github.com/basdenooijer/solarium/raw/master/COPYING
+ * @link http://www.solarium-project.org/
+ *
+ * @package Solarium
+ */
+
+/**
+ * Buffered add plugin
+ *
+ * If you need to add (or update) a big number of documents to Solr it's much more efficient to do so in batches.
+ * This plugin makes this as easy as possible.
+ *
+ * @package Solarium
+ * @subpackage Plugin
+ */
+class Solarium_Plugin_BufferedAdd extends Solarium_Plugin_Abstract
+{
+
+    /**
+     * Default options
+     *
+     * @var array
+     */
+    protected $_options = array(
+        'buffersize' => 100,
+    );
+
+    /**
+     * Update query instance
+     *
+     * @var Solarium_Query_Update
+     */
+    protected $_updateQuery;
+
+    /**
+     * Buffered documents
+     *
+     * @var array
+     */
+    protected $_buffer = array();
+
+    /**
+     * Plugin init function
+     *
+     * This is an extension point for plugin implementations.
+     * Will be called as soon as $this->_client and options have been set.
+     *
+     * @return void
+     */
+    protected function _initPlugin()
+    {
+        $this->_updateQuery = $this->_client->createUpdate();
+    }
+
+    /**
+     * Set buffer size option
+     *
+     * @param int $size
+     * @return Solarium_Configurable
+     */
+    public function setBufferSize($size)
+    {
+        return $this->_setOption('buffersize', $size);
+    }
+
+    /**
+     * Get buffer size option value
+     *
+     * @return int
+     */
+    public function getBufferSize()
+    {
+        return $this->getOption('buffersize');
+    }
+
+    /**
+     * Create a document object instance and add it to the buffer
+     *
+     * @param array $fields
+     * @param array $boosts
+     * @return self Provides fluent interface
+     */
+    public function createDocument($fields, $boosts = array())
+    {
+        $doc = $this->_updateQuery->createDocument($fields, $boosts);
+        $this->addDocument($doc);
+
+        return $this;
+    }
+
+    /**
+     * Add a document
+     *
+     * @param Solarium_Document_ReadOnly $document
+     * @return self Provides fluent interface
+     */
+    public function addDocument($document)
+    {
+        $this->_buffer[] = $document;
+        if (count($this->_buffer) == $this->_options['buffersize']) {
+            $this->flush();
+        }
+
+        return $this;
+    }
+
+    /**
+     * Add multiple documents
+     *
+     * @param array
+     * @return self Provides fluent interface
+     */
+    public function addDocuments($documents)
+    {
+        foreach ($documents as $document) {
+            $this->addDocument($document);
+        }
+
+        return $this;
+    }
+
+    /**
+     * Get all documents currently in the buffer
+     *
+     * Any previously flushed documents will not be included!
+     *
+     * @return array
+     */
+    public function getDocuments()
+    {
+        return $this->_buffer;
+    }
+
+    /**
+     * Clear any buffered documents
+     *
+     * @return self Provides fluent interface
+     */
+    public function clear()
+    {
+        $this->_updateQuery = $this->_client->createUpdate();
+        $this->_buffer = array();
+        return $this;
+    }
+
+    /**
+     * Flush any buffered documents to Solr
+     *
+     * @param boolean $overwrite
+     * @param int $commitWithin
+     * @return boolean|Solarium_Result_Update
+     */
+    public function flush($overwrite = null, $commitWithin = null)
+    {
+        if(count($this->_buffer) == 0) {
+            // nothing to do
+            return false;
+        }
+
+        $this->_client->triggerEvent('BufferedAddFlushStart', array($this->_buffer));
+
+        $this->_updateQuery->addDocuments($this->_buffer, $overwrite, $commitWithin);
+        $result = $this->_client->update($this->_updateQuery);
+        $this->clear();
+
+        $this->_client->triggerEvent('BufferedAddFlushEnd', array($result));
+
+        return $result;
+    }
+
+    /**
+     * Commit changes
+     *
+     * Any remaining documents in the buffer will also be flushed
+     *
+     * @param boolean $overwrite
+     * @param boolean $waitFlush
+     * @param boolean $waitSearcher
+     * @param boolean $expungeDeletes
+     * @return Solarium_Result_Update
+     */
+    public function commit($overwrite = null, $waitFlush = null, $waitSearcher = null, $expungeDeletes = null)
+    {
+        $this->_client->triggerEvent('BufferedAddCommitStart', array($this->_buffer));
+
+        $this->_updateQuery->addDocuments($this->_buffer, $overwrite);
+        $this->_updateQuery->addCommit($waitFlush, $waitSearcher, $expungeDeletes);
+        $result = $this->_client->update($this->_updateQuery);
+        $this->clear();
+
+        $this->_client->triggerEvent('BufferedAddCommitEnd', array($result));
+
+        return $result;
+    }
+
+}