Merge branch 'MDL-60174-master' of https://github.com/sammarshallou/m…

…oodle

admin/cli/fix_deleted_users.php

@@ -73,6 +73,7 @@
     echo "Redeleting user $user->id: $user->username ($user->email)\n";
     delete_user($user);
 }
+$rs->close();
 
 cli_heading('Deleting all leftovers');
 

admin/tool/messageinbound/index.php

@@ -40,6 +40,7 @@
     foreach ($records as $record) {
         $instances[] = \core\message\inbound\manager::get_handler($record->classname);
     }
+    $records->close();
 
     echo $OUTPUT->header();
     echo $renderer->messageinbound_handlers_table($instances);

admin/tool/spamcleaner/index.php

@@ -234,15 +234,19 @@ function search_spammers($keywords) {
     $keywordlist = implode(', ', $keywords);
     echo $OUTPUT->box(get_string('spamresult', 'tool_spamcleaner').s($keywordlist)).' ...';
 
-    print_user_list(array($spamusers_desc,
-                          $spamusers_blog,
-                          $spamusers_blogsub,
-                          $spamusers_comment,
-                          $spamusers_message,
-                          $spamusers_forumpost,
-                          $spamusers_forumpostsub
-                         ),
-                         $keywords);
+    $recordsets = [
+        $spamusers_desc,
+        $spamusers_blog,
+        $spamusers_blogsub,
+        $spamusers_comment,
+        $spamusers_message,
+        $spamusers_forumpost,
+        $spamusers_forumpostsub
+    ];
+    print_user_list($recordsets, $keywords);
+    foreach ($recordsets as $rs) {
+        $rs->close();
+    }
 }
 
 

analytics/classes/manager.php

@@ -360,6 +360,7 @@ public static function get_indicator_calculations($analysable, $starttime, $endt
             }
             $existingcalculations[$calculation->indicator][$calculation->sampleid] = $calculation->value;
         }
+        $calculations->close();
         return $existingcalculations;
     }
 

backup/moodle2/restore_stepslib.php

@@ -981,8 +981,8 @@ protected function define_execution() {
                 $DB->set_field('course_' . $table . 's', 'availability', $newvalue,
                         array('id' => $thingid));
             }
+            $rs->close();
         }
-        $rs->close();
     }
 }
 

cohort/lib.php

@@ -504,6 +504,7 @@ function cohort_get_invisible_contexts() {
             $excludedcontexts[] = $ctx->id;
         }
     }
+    $records->close();
     return $excludedcontexts;
 }
 

grade/grading/form/lib.php

@@ -428,6 +428,7 @@ public function get_active_instances($itemid) {
         foreach ($records as $record) {
             $rv[] = $this->get_instance($record);
         }
+        $records->close();
         return $rv;
     }
 

grade/report/singleview/classes/local/screen/screen.php

@@ -419,6 +419,7 @@ protected function load_users() {
         while ($user = $gui->next_user()) {
             $users[$user->user->id] = $user->user;
         }
+        $gui->close();
         return $users;
     }
 

group/lib.php

@@ -628,6 +628,7 @@ function groups_delete_groupings_groups($courseid, $showfeedback=false) {
     foreach ($results as $result) {
         groups_unassign_grouping($result->groupingid, $result->groupid, false);
     }
+    $results->close();
 
     // Invalidate the grouping cache for the course
     cache_helper::invalidate_by_definition('core', 'groupdata', array(), array($courseid));
@@ -658,6 +659,7 @@ function groups_delete_groups($courseid, $showfeedback=false) {
     foreach ($groups as $group) {
         groups_delete_group($group);
     }
+    $groups->close();
 
     // Invalidate the grouping cache for the course
     cache_helper::invalidate_by_definition('core', 'groupdata', array(), array($courseid));
@@ -690,6 +692,7 @@ function groups_delete_groupings($courseid, $showfeedback=false) {
     foreach ($groupings as $grouping) {
         groups_delete_grouping($grouping);
     }
+    $groupings->close();
 
     // Invalidate the grouping cache for the course.
     cache_helper::invalidate_by_definition('core', 'groupdata', array(), array($courseid));

lib/blocklib.php

@@ -800,6 +800,7 @@ public function load_blocks($includeinvisible = null) {
                 $unknown[] = $bi;
             }
         }
+        $blockinstances->close();
 
         // Pages don't necessarily have a defaultregion. The  one time this can
         // happen is when there are no theme block regions, but the script itself

lib/classes/message/inbound/manager.php

@@ -74,6 +74,7 @@ public static function update_handlers_for_component($componentname) {
                 self::remove_messageinbound_handler($handler);
             }
         }
+        $existinghandlers->close();
 
         self::create_missing_messageinbound_handlers_for_component($componentname);
     }

lib/classes/plugininfo/portfolio.php

@@ -43,6 +43,7 @@ public static function get_enabled_plugins() {
         foreach ($rs as $repository) {
             $enabled[$repository->plugin] = $repository->plugin;
         }
+        $rs->close();
 
         return $enabled;
     }
@@ -91,4 +92,4 @@ public function uninstall_cleanup() {
 
         parent::uninstall_cleanup();
     }
-}
+}

lib/db/upgrade.php

@@ -1183,6 +1183,7 @@ function xmldb_main_upgrade($oldversion) {
             $i++;
             $pbar->update($i, $total, "Updating duplicate question category stamp - $i/$total.");
         }
+        $rs->close();
         unset($usedstamps);
 
         // The uniqueness of each (contextid, stamp) pair is now guaranteed, so add the unique index to stop future duplicates.

lib/dml/pgsql_native_moodle_database.php

@@ -45,6 +45,12 @@ class pgsql_native_moodle_database extends moodle_database {
     /** @var bool savepoint hack for MDL-35506 - workaround for automatic transaction rollback on error */
     protected $savepointpresent = false;
 
+    /** @var int Number of cursors used (for constructing a unique ID) */
+    protected $cursorcount = 0;
+
+    /** @var int Default number of rows to fetch at a time when using recordsets with cursors */
+    const DEFAULT_FETCH_BUFFER_SIZE = 100000;
+
     /**
      * Detects if all needed PHP stuff installed.
      * Note: can be used before connect()
@@ -734,14 +740,89 @@ public function get_recordset_sql($sql, array $params=null, $limitfrom=0, $limit
         list($sql, $params, $type) = $this->fix_sql_params($sql, $params);
 
         $this->query_start($sql, $params, SQL_QUERY_SELECT);
-        $result = pg_query_params($this->pgsql, $sql, $params);
+
+        // For any query that doesn't explicitly specify a limit, we must use cursors to stop it
+        // loading the entire thing (unless the config setting is turned off).
+        $usecursors = !$limitnum && ($this->get_fetch_buffer_size() > 0);
+        if ($usecursors) {
+            // Work out the cursor unique identifer. This is based on a simple count used which
+            // should be OK because the identifiers only need to be unique within the current
+            // transaction.
+            $this->cursorcount++;
+            $cursorname = 'crs' . $this->cursorcount;
+
+            // Do the query to a cursor.
+            $sql = 'DECLARE ' . $cursorname . ' NO SCROLL CURSOR WITH HOLD FOR ' . $sql;
+            $result = pg_query_params($this->pgsql, $sql, $params);
+        } else {
+            $result = pg_query_params($this->pgsql, $sql, $params);
+            $cursorname = '';
+        }
+
         $this->query_end($result);
+        if ($usecursors) {
+            pg_free_result($result);
+            $result = null;
+        }
 
-        return $this->create_recordset($result);
+        return new pgsql_native_moodle_recordset($result, $this, $cursorname);
     }
 
-    protected function create_recordset($result) {
-        return new pgsql_native_moodle_recordset($result);
+    /**
+     * Gets size of fetch buffer used for recordset queries.
+     *
+     * If this returns 0 then cursors will not be used, meaning recordset queries will occupy enough
+     * memory as needed for the Postgres library to hold the entire query results in memory.
+     *
+     * @return int Fetch buffer size or 0 indicating not to use cursors
+     */
+    protected function get_fetch_buffer_size() {
+        if (array_key_exists('fetchbuffersize', $this->dboptions)) {
+            return (int)$this->dboptions['fetchbuffersize'];
+        } else {
+            return self::DEFAULT_FETCH_BUFFER_SIZE;
+        }
+    }
+
+    /**
+     * Retrieves data from cursor. For use by recordset only; do not call directly.
+     *
+     * Return value contains the next batch of Postgres data, and a boolean indicating if this is
+     * definitely the last batch (if false, there may be more)
+     *
+     * @param string $cursorname Name of cursor to read from
+     * @return array Array with 2 elements (next data batch and boolean indicating last batch)
+     */
+    public function fetch_from_cursor($cursorname) {
+        $count = $this->get_fetch_buffer_size();
+
+        $sql = 'FETCH ' . $count . ' FROM ' . $cursorname;
+
+        $this->query_start($sql, [], SQL_QUERY_AUX);
+        $result = pg_query($this->pgsql, $sql);
+        $last = pg_num_rows($result) !== $count;
+
+        $this->query_end($result);
+
+        return [$result, $last];
+    }
+
+    /**
+     * Closes a cursor. For use by recordset only; do not call directly.
+     *
+     * @param string $cursorname Name of cursor to close
+     * @return bool True if we actually closed one, false if the transaction was cancelled
+     */
+    public function close_cursor($cursorname) {
+        // If the transaction got cancelled, then ignore this request.
+        $sql = 'CLOSE ' . $cursorname;
+        $this->query_start($sql, [], SQL_QUERY_AUX);
+        $result = pg_query($this->pgsql, $sql);
+        $this->query_end($result);
+        if ($result) {
+            pg_free_result($result);
+        }
+        return true;
     }
 
     /**
@@ -1366,7 +1447,7 @@ public function release_session_lock($rowid) {
     protected function begin_transaction() {
         $this->savepointpresent = true;
         $sql = "BEGIN ISOLATION LEVEL READ COMMITTED; SAVEPOINT moodle_pg_savepoint";
-        $this->query_start($sql, NULL, SQL_QUERY_AUX);
+        $this->query_start($sql, null, SQL_QUERY_AUX);
         $result = pg_query($this->pgsql, $sql);
         $this->query_end($result);
 
@@ -1381,7 +1462,7 @@ protected function begin_transaction() {
     protected function commit_transaction() {
         $this->savepointpresent = false;
         $sql = "RELEASE SAVEPOINT moodle_pg_savepoint; COMMIT";
-        $this->query_start($sql, NULL, SQL_QUERY_AUX);
+        $this->query_start($sql, null, SQL_QUERY_AUX);
         $result = pg_query($this->pgsql, $sql);
         $this->query_end($result);
 
@@ -1396,7 +1477,7 @@ protected function commit_transaction() {
     protected function rollback_transaction() {
         $this->savepointpresent = false;
         $sql = "RELEASE SAVEPOINT moodle_pg_savepoint; ROLLBACK";
-        $this->query_start($sql, NULL, SQL_QUERY_AUX);
+        $this->query_start($sql, null, SQL_QUERY_AUX);
         $result = pg_query($this->pgsql, $sql);
         $this->query_end($result);
 

lib/dml/pgsql_native_moodle_recordset.php

@@ -40,26 +40,64 @@ class pgsql_native_moodle_recordset extends moodle_recordset {
     protected $current;
     protected $blobs = array();
 
+    /** @var string Name of cursor or '' if none */
+    protected $cursorname;
+
+    /** @var pgsql_native_moodle_database Postgres database resource */
+    protected $db;
+
+    /** @var bool True if there are no more rows to fetch from the cursor */
+    protected $lastbatch;
+
     /**
      * Build a new recordset to iterate over.
      *
-     * @param resource $result A pg_query() result object to create a recordset from.
+     * When using cursors, $result will be null initially.
+     *
+     * @param resource|null $result A pg_query() result object to create a recordset from.
+     * @param pgsql_native_moodle_database $db Database object (only required when using cursors)
+     * @param string $cursorname Name of cursor or '' if none
      */
-    public function __construct($result) {
+    public function __construct($result, pgsql_native_moodle_database $db = null, $cursorname = '') {
+        if ($cursorname && !$db) {
+            throw new coding_exception('When specifying a cursor, $db is required');
+        }
         $this->result = $result;
+        $this->db = $db;
+        $this->cursorname = $cursorname;
+
+        // When there is a cursor, do the initial fetch.
+        if ($cursorname) {
+            $this->fetch_cursor_block();
+        }
 
         // Find out if there are any blobs.
-        $numfields = pg_num_fields($result);
+        $numfields = pg_num_fields($this->result);
         for ($i = 0; $i < $numfields; $i++) {
-            $type = pg_field_type($result, $i);
+            $type = pg_field_type($this->result, $i);
             if ($type == 'bytea') {
-                $this->blobs[] = pg_field_name($result, $i);
+                $this->blobs[] = pg_field_name($this->result, $i);
             }
         }
 
         $this->current = $this->fetch_next();
     }
 
+    /**
+     * Fetches the next block of data when using cursors.
+     *
+     * @throws coding_exception If you call this when the fetch buffer wasn't freed yet
+     */
+    protected function fetch_cursor_block() {
+        if ($this->result) {
+            throw new coding_exception('Unexpected non-empty result when fetching from cursor');
+        }
+        list($this->result, $this->lastbatch) = $this->db->fetch_from_cursor($this->cursorname);
+        if (!$this->result) {
+            throw new coding_exception('Unexpected failure when fetching from cursor');
+        }
+    }
+
     public function __destruct() {
         $this->close();
     }
@@ -69,9 +107,21 @@ private function fetch_next() {
             return false;
         }
         if (!$row = pg_fetch_assoc($this->result)) {
+            // There are no more rows in this result.
             pg_free_result($this->result);
             $this->result = null;
-            return false;
+
+            // If using a cursor, can we fetch the next block?
+            if ($this->cursorname && !$this->lastbatch) {
+                $this->fetch_cursor_block();
+                if (!$row = pg_fetch_assoc($this->result)) {
+                    pg_free_result($this->result);
+                    $this->result = null;
+                    return false;
+                }
+            } else {
+                return false;
+            }
         }
 
         if ($this->blobs) {
@@ -111,5 +161,11 @@ public function close() {
         }
         $this->current = null;
         $this->blobs   = null;
+
+        // If using cursors, close the cursor.
+        if ($this->cursorname) {
+            $this->db->close_cursor($this->cursorname);
+            $this->cursorname = null;
+        }
     }
 }