MDL-48595 logstore: Adding new interfaces

sql_reader and sql_internal_table_reader.

admin/tool/log/store/database/classes/log/store.php

@@ -25,7 +25,7 @@
 namespace logstore_database\log;
 defined('MOODLE_INTERNAL') || die();
 
-class store implements \tool_log\log\writer, \core\log\sql_select_reader {
+class store implements \tool_log\log\writer, \core\log\sql_reader {
     use \tool_log\helper\store,
         \tool_log\helper\reader,
         \tool_log\helper\buffered_writer {
@@ -173,28 +173,71 @@ public function get_events_select($selectwhere, array $params, $sort, $limitfrom
         $records = $this->extdb->get_records_select($dbtable, $selectwhere, $params, $sort, '*', $limitfrom, $limitnum);
 
         foreach ($records as $data) {
-            $extra = array('origin' => $data->origin, 'realuserid' => $data->realuserid, 'ip' => $data->ip);
-            $data = (array)$data;
-            $id = $data['id'];
-            $data['other'] = unserialize($data['other']);
-            if ($data['other'] === false) {
-                $data['other'] = array();
-            }
-            unset($data['origin']);
-            unset($data['ip']);
-            unset($data['realuserid']);
-            unset($data['id']);
-
-            $event = \core\event\base::restore($data, $extra);
-            // Add event to list if it's valid.
-            if ($event) {
-                $events[$id] = $event;
+            if ($event = $this->get_log_event($data)) {
+                $events[$data->id] = $event;
             }
         }
 
         return $events;
     }
 
+    /**
+     * Fetch records using given criteria returning a Traversable object.
+     *
+     * Note that the traversable object contains a moodle_recordset, so
+     * remember that is important that you call close() once you finish
+     * using it.
+     *
+     * @param string $selectwhere
+     * @param array $params
+     * @param string $sort
+     * @param int $limitfrom
+     * @param int $limitnum
+     * @return \core\dml\recordset_walk|\core\event\base[]
+     */
+    public function get_events_select_iterator($selectwhere, array $params, $sort, $limitfrom, $limitnum) {
+        if (!$this->init()) {
+            return array();
+        }
+
+        if (!$dbtable = $this->get_config('dbtable')) {
+            return array();
+        }
+
+        $sort = self::tweak_sort_by_id($sort);
+
+        $recordset = $this->extdb->get_recordset_select($dbtable, $selectwhere, $params, $sort, '*', $limitfrom, $limitnum);
+
+        return new \core\dml\recordset_walk($recordset, array($this, 'get_log_event'));
+    }
+
+    /**
+     * Returns an event from the log data.
+     *
+     * @param stdClass $data Log data
+     * @return \core\event\base
+     */
+    public function get_log_event($data) {
+
+        $extra = array('origin' => $data->origin, 'ip' => $data->ip, 'realuserid' => $data->realuserid);
+        $data = (array)$data;
+        $id = $data['id'];
+        $data['other'] = unserialize($data['other']);
+        if ($data['other'] === false) {
+            $data['other'] = array();
+        }
+        unset($data['origin']);
+        unset($data['ip']);
+        unset($data['realuserid']);
+        unset($data['id']);
+
+        if (!$event = \core\event\base::restore($data, $extra)) {
+            return null;
+        }
+
+        return $event;
+    }
+
     /**
      * Get number of events present for the given select clause.
      *

admin/tool/log/store/legacy/classes/log/store.php

@@ -26,7 +26,7 @@
 
 defined('MOODLE_INTERNAL') || die();
 
-class store implements \tool_log\log\store, \core\log\sql_select_reader {
+class store implements \tool_log\log\store, \core\log\sql_reader {
     use \tool_log\helper\store,
         \tool_log\helper\reader;
 
@@ -47,7 +47,7 @@ public function __construct(\tool_log\log\manager $manager) {
     const CRUD_REGEX = "/(crud).*?(<>|=|!=).*?'(.*?)'/s";
 
     /**
-     * This method contains mapping required for Moodle core to make legacy store compatible with other sql_select_reader based
+     * This method contains mapping required for Moodle core to make legacy store compatible with other sql_reader based
      * queries.
      *
      * @param string $selectwhere Select statment
@@ -104,14 +104,56 @@ public function get_events_select($selectwhere, array $params, $sort, $limitfrom
         $events = array();
 
         foreach ($records as $data) {
-            $events[$data->id] = \logstore_legacy\event\legacy_logged::restore_legacy($data);
+            $events[$data->id] = $this->get_log_event($data);
         }
 
         $records->close();
 
         return $events;
     }
 
+    /**
+     * Fetch records using given criteria returning a Traversable object.
+     *
+     * Note that the traversable object contains a moodle_recordset, so
+     * remember that is important that you call close() once you finish
+     * using it.
+     *
+     * @param string $selectwhere
+     * @param array $params
+     * @param string $sort
+     * @param int $limitfrom
+     * @param int $limitnum
+     * @return \Traversable|\core\event\base[]
+     */
+    public function get_events_select_iterator($selectwhere, array $params, $sort, $limitfrom, $limitnum) {
+        global $DB;
+
+        $sort = self::tweak_sort_by_id($sort);
+
+        // Replace the query with hardcoded mappings required for core.
+        list($selectwhere, $params, $sort) = self::replace_sql_legacy($selectwhere, $params, $sort);
+
+        try {
+            $recordset = $DB->get_recordset_select('log', $selectwhere, $params, $sort, '*', $limitfrom, $limitnum);
+        } catch (\moodle_exception $ex) {
+            debugging("error converting legacy event data " . $ex->getMessage() . $ex->debuginfo, DEBUG_DEVELOPER);
+            return new \EmptyIterator;
+        }
+
+        return new \core\dml\recordset_walk($recordset, array($this, 'get_log_event'));
+    }
+
+    /**
+     * Returns an event from the log data.
+     *
+     * @param stdClass $data Log data
+     * @return \core\event\base
+     */
+    public function get_log_event($data) {
+        return \logstore_legacy\event\legacy_logged::restore_legacy($data);
+    }
+
     public function get_events_select_count($selectwhere, array $params) {
         global $DB;
 

admin/tool/log/store/legacy/tests/store_test.php

@@ -50,7 +50,7 @@ public function test_log_writing() {
         $this->assertEquals(array('logstore_legacy'), array_keys($stores));
         $store = $stores['logstore_legacy'];
         $this->assertInstanceOf('logstore_legacy\log\store', $store);
-        $this->assertInstanceOf('core\log\sql_select_reader', $store);
+        $this->assertInstanceOf('core\log\sql_reader', $store);
         $this->assertTrue($store->is_logging());
 
         $logs = $DB->get_records('log', array(), 'id ASC');

admin/tool/log/store/standard/classes/log/store.php

@@ -26,7 +26,7 @@
 
 defined('MOODLE_INTERNAL') || die();
 
-class store implements \tool_log\log\writer, \core\log\sql_internal_reader {
+class store implements \tool_log\log\writer, \core\log\sql_internal_table_reader {
     use \tool_log\helper\store,
         \tool_log\helper\buffered_writer,
         \tool_log\helper\reader;
@@ -75,22 +75,8 @@ public function get_events_select($selectwhere, array $params, $sort, $limitfrom
         $records = $DB->get_recordset_select('logstore_standard_log', $selectwhere, $params, $sort, '*', $limitfrom, $limitnum);
 
         foreach ($records as $data) {
-            $extra = array('origin' => $data->origin, 'ip' => $data->ip, 'realuserid' => $data->realuserid);
-            $data = (array)$data;
-            $id = $data['id'];
-            $data['other'] = unserialize($data['other']);
-            if ($data['other'] === false) {
-                $data['other'] = array();
-            }
-            unset($data['origin']);
-            unset($data['ip']);
-            unset($data['realuserid']);
-            unset($data['id']);
-
-            $event = \core\event\base::restore($data, $extra);
-            // Add event to list if it's valid.
-            if ($event) {
-                $events[$id] = $event;
+            if ($event = $this->get_log_event($data)) {
+                $events[$data->id] = $event;
             }
         }
 
@@ -99,6 +85,57 @@ public function get_events_select($selectwhere, array $params, $sort, $limitfrom
         return $events;
     }
 
+    /**
+     * Fetch records using given criteria returning a Traversable object.
+     *
+     * Note that the traversable object contains a moodle_recordset, so
+     * remember that is important that you call close() once you finish
+     * using it.
+     *
+     * @param string $selectwhere
+     * @param array $params
+     * @param string $sort
+     * @param int $limitfrom
+     * @param int $limitnum
+     * @return \core\dml\recordset_walk|\core\event\base[]
+     */
+    public function get_events_select_iterator($selectwhere, array $params, $sort, $limitfrom, $limitnum) {
+        global $DB;
+
+        $sort = self::tweak_sort_by_id($sort);
+
+        $recordset = $DB->get_recordset_select('logstore_standard_log', $selectwhere, $params, $sort, '*', $limitfrom, $limitnum);
+
+        return new \core\dml\recordset_walk($recordset, array($this, 'get_log_event'));
+    }
+
+    /**
+     * Returns an event from the log data.
+     *
+     * @param stdClass $data Log data
+     * @return \core\event\base
+     */
+    public function get_log_event($data) {
+
+        $extra = array('origin' => $data->origin, 'ip' => $data->ip, 'realuserid' => $data->realuserid);
+        $data = (array)$data;
+        $id = $data['id'];
+        $data['other'] = unserialize($data['other']);
+        if ($data['other'] === false) {
+            $data['other'] = array();
+        }
+        unset($data['origin']);
+        unset($data['ip']);
+        unset($data['realuserid']);
+        unset($data['id']);
+
+        if (!$event = \core\event\base::restore($data, $extra)) {
+            return null;
+        }
+
+        return $event;
+    }
+
     public function get_events_select_count($selectwhere, array $params) {
         global $DB;
         return $DB->count_records_select('logstore_standard_log', $selectwhere, $params);

admin/tool/log/store/standard/tests/store_test.php

@@ -218,4 +218,59 @@ public function test_get_supported_reports() {
             $this->assertContains($expectedreport, $reports);
         }
     }
+
+    /**
+     * Test sql_reader::get_events_select_iterator.
+     * @return void
+     */
+    public function test_events_traversable() {
+        global $DB;
+
+        $this->resetAfterTest();
+        $this->preventResetByRollback();
+        $this->setAdminUser();
+
+        set_config('enabled_stores', 'logstore_standard', 'tool_log');
+
+        $manager = get_log_manager(true);
+        $stores = $manager->get_readers();
+        $store = $stores['logstore_standard'];
+
+        $events = $store->get_events_select_iterator('', array(), '', 0, 0);
+        $this->assertFalse($events->valid());
+
+        // Here it should be already closed, but we should be allowed to
+        // over-close it without exception.
+        $events->close();
+
+        $user = $this->getDataGenerator()->create_user();
+        for ($i = 0; $i < 1000; $i++) {
+            \core\event\user_created::create_from_userid($user->id)->trigger();
+        }
+        $store->flush();
+
+        // Check some various sizes get the right number of elements.
+        $this->assertEquals(1, iterator_count($store->get_events_select_iterator('', array(), '', 0, 1)));
+        $this->assertEquals(2, iterator_count($store->get_events_select_iterator('', array(), '', 0, 2)));
+
+        $iterator = $store->get_events_select_iterator('', array(), '', 0, 500);
+        $this->assertInstanceOf('\core\event\base', $iterator->current());
+        $this->assertEquals(500, iterator_count($iterator));
+        $iterator->close();
+
+        // Look for non-linear memory usage for the iterator version.
+        $mem = memory_get_usage();
+        $events = $store->get_events_select('', array(), '', 0, 0);
+        $delta1 = memory_get_usage() - $mem;
+        $events = $store->get_events_select_iterator('', array(), '', 0, 0);
+        $delta2 = memory_get_usage() - $mem;
+        $this->assertInstanceOf('\Traversable', $events);
+        $events->close();
+
+        $this->assertLessThan($delta1 / 10, $delta2);
+
+        set_config('enabled_stores', '', 'tool_log');
+        get_log_manager(true);
+    }
+
 }

lib/tablelib.php

@@ -1345,7 +1345,7 @@ class table_sql extends flexible_table {
      */
     public $sql = NULL;
     /**
-     * @var array Data fetched from the db.
+     * @var array|\Traversable Data fetched from the db.
      */
     public $rawdata = NULL;
 
@@ -1374,14 +1374,27 @@ function __construct($uniqueid) {
      * processing each col using either col_{columnname} method or other_cols
      * method or if other_cols returns NULL then put the data straight into the
      * table.
+     *
+     * @return void
      */
     function build_table() {
-        if ($this->rawdata) {
-            foreach ($this->rawdata as $row) {
-                $formattedrow = $this->format_row($row);
-                $this->add_data_keyed($formattedrow,
-                        $this->get_row_class($row));
-            }
+
+        if ($this->rawdata instanceof \Traversable && !$this->rawdata->valid()) {
+            return;
+        }
+        if (!$this->rawdata) {
+            return;
+        }
+
+        foreach ($this->rawdata as $row) {
+            $formattedrow = $this->format_row($row);
+            $this->add_data_keyed($formattedrow,
+                $this->get_row_class($row));
+        }
+
+        if ($this->rawdata instanceof \core\dml\recordset_walk ||
+                $this->rawdata instanceof moodle_recordset) {
+            $this->rawdata->close();
         }
     }