MDL-58885 core_search: Add group support

Adds group support to the core search API and the Solr search engine.
This allows for:

* User searching by group (in the API only, no interface yet)
* Automatically restrict search results by group (in some cases like
  separate-groups forums)

lang/en/search.php

@@ -90,6 +90,8 @@
 $string['resultsreturnedfor'] = 'results returned for';
 $string['runindexer'] = 'Run indexer (real)';
 $string['runindexertest'] = 'Run indexer test';
+$string['schemanotupdated'] = 'The search schema is out of date.';
+$string['schemaversionunknown'] = 'Search engine does not know about the current schema version.';
 $string['score'] = 'Score';
 $string['search'] = 'Search';
 $string['search:message_received'] = 'Messages - received';

search/classes/base_mod.php

@@ -245,4 +245,45 @@ public function get_contexts_to_reindex() {
             return \context::instance_by_id($id);
         });
     }
+
+    /**
+     * Indicates whether this search area may restrict access by group.
+     *
+     * This should return true if the search area (sometimes) sets the 'groupid' schema field, and
+     * false if it never sets that field.
+     *
+     * (If this function returns false, but the field is set, then results may be restricted
+     * unintentionally.)
+     *
+     * If this returns true, the search engine will automatically apply group restrictions in some
+     * cases (by default, where a module is configured to use separate groups). See function
+     * restrict_cm_access_by_group().
+     *
+     * @return bool
+     */
+    public function supports_group_restriction() {
+        return false;
+    }
+
+    /**
+     * Checks whether the content of this search area should be restricted by group for a
+     * specific module. Called at query time.
+     *
+     * The default behaviour simply checks if the effective group mode is SEPARATEGROUPS, which
+     * is probably correct for most cases.
+     *
+     * If restricted by group, the search query will (where supported by the engine) filter out
+     * results for groups the user does not belong to, unless the user has 'access all groups'
+     * for the activity. This affects only documents which set the 'groupid' field; results with no
+     * groupid will not be restricted.
+     *
+     * Even if you return true to this function, you may still need to do group access checks in
+     * check_access, because the search engine may not support group restrictions.
+     *
+     * @param \cm_info $cm
+     * @return bool True to restrict by group
+     */
+    public function restrict_cm_access_by_group(\cm_info $cm) {
+        return $cm->effectivegroupmode == SEPARATEGROUPS;
+    }
 }

search/classes/document.php

@@ -78,6 +78,14 @@ class document implements \renderable, \templatable {
      */
     protected $files = array();
 
+    /**
+     * Change list (for engine implementers):
+     * 2017091700 - add optional field groupid
+     *
+     * @var int Schema version number (update if any change)
+     */
+    const SCHEMA_VERSION = 2017091700;
+
     /**
      * All required fields any doc should contain.
      *
@@ -159,6 +167,11 @@ class document implements \renderable, \templatable {
             'stored' => true,
             'indexed' => true
         ),
+        'groupid' => array(
+            'type' => 'int',
+            'stored' => true,
+            'indexed' => true
+        ),
         'description1' => array(
             'type' => 'text',
             'stored' => true,

search/classes/engine.php

@@ -42,7 +42,7 @@ abstract class engine {
     /**
      * The search engine configuration.
      *
-     * @var stdClass
+     * @var \stdClass
      */
     protected $config = null;
 
@@ -65,7 +65,7 @@ abstract class engine {
     /**
      * User data required to show their fullnames. Indexed by userid.
      *
-     * @var stdClass[]
+     * @var \stdClass[]
      */
     protected static $cachedusers = array();
 
@@ -439,12 +439,36 @@ abstract function add_document($document, $fileindexing = false);
      *
      * Engines should reasonably attempt to fill up to limit with valid results if they are available.
      *
-     * @param  stdClass $filters Query and filters to apply.
-     * @param  array    $usercontexts Contexts where the user has access. True if the user can access all contexts.
+     * The $filters object may include the following fields (optional except q):
+     * - q: value of main search field; results should include this text
+     * - title: if included, title must match this search
+     * - areaids: array of search area id strings (only these areas will be searched)
+     * - courseids: array of course ids (only these courses will be searched)
+     * - groupids: array of group ids (only results specifically from these groupids will be
+     *   searched) - this option will be ignored if the search engine doesn't support groups
+     *
+     * The $accessinfo parameter has two different values (for historical compatibility). If the
+     * engine returns false to supports_group_filtering then it is an array of user contexts, or
+     * true if the user can access all contexts. (This parameter used to be called $usercontexts.)
+     * If the engine returns true to supports_group_filtering then it will be an object containing
+     * these fields:
+     * - everything (true if admin is searching with no restrictions)
+     * - usercontexts (same as above)
+     * - separategroupscontexts (array of context ids where separate groups are used)
+     * - visiblegroupscontextsareas (array of subset of those where some areas use visible groups)
+     * - usergroups (array of relevant group ids that user belongs to)
+     *
+     * The engine should apply group restrictions to those contexts listed in the
+     * 'separategroupscontexts' array. In these contexts, it shouled only include results if the
+     * groupid is not set, or if the groupid matches one of the values in USER_GROUPS array, or
+     * if the search area is one of those listed in 'visiblegroupscontextsareas' for that context.
+     *
+     * @param \stdClass $filters Query and filters to apply.
+     * @param \stdClass $accessinfo Information about the contexts the user can access
      * @param  int      $limit The maximum number of results to return. If empty, limit to manager::MAX_RESULTS.
      * @return \core_search\document[] Results or false if no results
      */
-    abstract function execute_query($filters, $usercontexts, $limit = 0);
+    public abstract function execute_query($filters, $accessinfo, $limit = 0);
 
     /**
      * Delete all documents.
@@ -453,4 +477,70 @@ abstract function execute_query($filters, $usercontexts, $limit = 0);
      * @return void
      */
     abstract function delete($areaid = null);
+
+    /**
+     * Checks that the schema is the latest version. If the version stored in config does not match
+     * the current, this function will attempt to upgrade the schema.
+     *
+     * @return bool|string True if schema is OK, a string if user needs to take action
+     */
+    public function check_latest_schema() {
+        if (empty($this->config->schemaversion)) {
+            $currentversion = 0;
+        } else {
+            $currentversion = $this->config->schemaversion;
+        }
+        if ($currentversion < document::SCHEMA_VERSION) {
+            return $this->update_schema((int)$currentversion, (int)document::SCHEMA_VERSION);
+        } else {
+            return true;
+        }
+    }
+
+    /**
+     * Usually called by the engine; marks that the schema has been updated.
+     *
+     * @param int $version Records the schema version now applied
+     */
+    public function record_applied_schema_version($version) {
+        set_config('schemaversion', $version, $this->pluginname);
+    }
+
+    /**
+     * Requests the search engine to upgrade the schema. The engine should update the schema if
+     * possible/necessary, and should ensure that record_applied_schema_version is called as a
+     * result.
+     *
+     * If it is not possible to upgrade the schema at the moment, it can do nothing and return; the
+     * function will be called again next time search is initialised.
+     *
+     * The default implementation just returns, with a DEBUG_DEVELOPER warning.
+     *
+     * @param int $oldversion Old schema version
+     * @param int $newversion New schema version
+     * @return bool|string True if schema is updated successfully, a string if it needs updating manually
+     */
+    protected function update_schema($oldversion, $newversion) {
+        debugging('Unable to update search engine schema: ' . $this->pluginname, DEBUG_DEVELOPER);
+        return get_string('schemanotupdated', 'search');
+    }
+
+    /**
+     * Checks if this search engine supports groups.
+     *
+     * Note that returning true to this function causes the parameters to execute_query to be
+     * passed differently!
+     *
+     * In order to implement groups and return true to this function, the search engine should:
+     *
+     * 1. Handle the fields ->separategroupscontexts and ->usergroups in the $accessinfo parameter
+     *    to execute_query (ideally, using these to automatically restrict search results).
+     * 2. Support the optional groupids parameter in the $filter parameter for execute_query to
+     *    restrict results to only those where the stored groupid matches the given value.
+     *
+     * @return bool True if this engine supports searching by group id field
+     */
+    public function supports_group_filtering() {
+        return false;
+    }
 }

search/classes/manager.php

@@ -364,11 +364,15 @@ public static function extract_areaid_parts($areaid) {
     }
 
     /**
-     * Returns the contexts the user can access.
+     * Returns information about the areas which the user can access.
      *
-     * The returned value is a multidimensional array because some search engines can group
-     * information and there will be a performance benefit on passing only some contexts
-     * instead of the whole context array set.
+     * The returned value is a stdClass object with the following fields:
+     * - everything (bool, true for admin only)
+     * - usercontexts (indexed by area identifier then context
+     * - separategroupscontexts (contexts within which group restrictions apply)
+     * - visiblegroupscontextsareas (overrides to the above when the same contexts also have
+     *   'visible groups' for certain search area ids - hopefully rare)
+     * - usergroups (groups which the current user belongs to)
      *
      * The areas can be limited by course id and context id. If specifying context ids, results
      * are limited to the exact context ids specified and not their children (for example, giving
@@ -378,15 +382,15 @@ public static function extract_areaid_parts($areaid) {
      *
      * @param array|false $limitcourseids An array of course ids to limit the search to. False for no limiting.
      * @param array|false $limitcontextids An array of context ids to limit the search to. False for no limiting.
-     * @return bool|array Indexed by area identifier (component + area name). Returns true if the user can see everything.
+     * @return \stdClass Object as described above
      */
     protected function get_areas_user_accesses($limitcourseids = false, $limitcontextids = false) {
         global $DB, $USER;
 
         // All results for admins (unless they have chosen to limit results). Eventually we could
         // add a new capability for managers.
         if (is_siteadmin() && !$limitcourseids && !$limitcontextids) {
-            return true;
+            return (object)array('everything' => true);
         }
 
         $areasbylevel = array();
@@ -404,6 +408,11 @@ protected function get_areas_user_accesses($limitcourseids = false, $limitcontex
         // This will store area - allowed contexts relations.
         $areascontexts = array();
 
+        // Initialise two special-case arrays for storing other information related to the contexts.
+        $separategroupscontexts = array();
+        $visiblegroupscontextsareas = array();
+        $usergroups = array();
+
         if (empty($limitcourseids) && !empty($areasbylevel[CONTEXT_SYSTEM])) {
             // We add system context to all search areas working at this level. Here each area is fully responsible of
             // the access control as we can not automate much, we can not even check guest access as some areas might
@@ -453,6 +462,7 @@ protected function get_areas_user_accesses($limitcourseids = false, $limitcontex
 
         // Keep a list of included course context ids (needed for the block calculation below).
         $coursecontextids = [];
+        $modulecms = [];
 
         foreach ($courses as $course) {
             if (!empty($limitcourseids) && !in_array($course->id, $limitcourseids)) {
@@ -462,6 +472,7 @@ protected function get_areas_user_accesses($limitcourseids = false, $limitcontex
 
             $coursecontext = \context_course::instance($course->id);
             $coursecontextids[] = $coursecontext->id;
+            $hasgrouprestrictions = false;
 
             // Info about the course modules.
             $modinfo = get_fast_modinfo($course);
@@ -491,11 +502,47 @@ protected function get_areas_user_accesses($limitcourseids = false, $limitcontex
                             continue;
                         }
                         if ($modinstance->uservisible) {
-                            $areascontexts[$areaid][$modinstance->context->id] = $modinstance->context->id;
+                            $contextid = $modinstance->context->id;
+                            $areascontexts[$areaid][$contextid] = $contextid;
+                            $modulecms[$modinstance->id] = $modinstance;
+
+                            if (!has_capability('moodle/site:accessallgroups', $modinstance->context) &&
+                                    ($searchclass instanceof base_mod) &&
+                                    $searchclass->supports_group_restriction()) {
+                                if ($searchclass->restrict_cm_access_by_group($modinstance)) {
+                                    $separategroupscontexts[$contextid] = $contextid;
+                                    $hasgrouprestrictions = true;
+                                } else {
+                                    // Track a list of anything that has a group id (so might get
+                                    // filtered) and doesn't want to be, in this context.
+                                    if (!array_key_exists($contextid, $visiblegroupscontextsareas)) {
+                                        $visiblegroupscontextsareas[$contextid] = array();
+                                    }
+                                    $visiblegroupscontextsareas[$contextid][$areaid] = $areaid;
+                                }
+                            }
                         }
                     }
                 }
             }
+
+            // Insert group information for course (unless there aren't any modules restricted by
+            // group for this user in this course, in which case don't bother).
+            if ($hasgrouprestrictions) {
+                $groups = groups_get_all_groups($course->id, $USER->id, 0, 'g.id');
+                foreach ($groups as $group) {
+                    $usergroups[$group->id] = $group->id;
+                }
+            }
+        }
+
+        // Chuck away all the 'visible groups contexts' data unless there is actually something
+        // that does use separate groups in the same context (this data is only used as an
+        // 'override' in cases where the search is restricting to separate groups).
+        foreach ($visiblegroupscontextsareas as $contextid => $areas) {
+            if (!array_key_exists($contextid, $separategroupscontexts)) {
+                unset($visiblegroupscontextsareas[$contextid]);
+            }
         }
 
         // Add all supported block contexts, in a single query for performance.
@@ -564,7 +611,10 @@ protected function get_areas_user_accesses($limitcourseids = false, $limitcontex
             }
         }
 
-        return $areascontexts;
+        // Return all the data.
+        return (object)array('everything' => false, 'usercontexts' => $areascontexts,
+                'separategroupscontexts' => $separategroupscontexts, 'usergroups' => $usergroups,
+                'visiblegroupscontextsareas' => $visiblegroupscontextsareas);
     }
 
     /**
@@ -687,12 +737,19 @@ public function search(\stdClass $formdata, $limit = 0) {
         // Clears previous query errors.
         $this->engine->clear_query_error();
 
-        $areascontexts = $this->get_areas_user_accesses($limitcourseids, $limitcontextids);
-        if (!$areascontexts) {
+        $contextinfo = $this->get_areas_user_accesses($limitcourseids, $limitcontextids);
+        if (!$contextinfo->everything && !$contextinfo->usercontexts) {
             // User can not access any context.
             $docs = array();
         } else {
-            $docs = $this->engine->execute_query($formdata, $areascontexts, $limit);
+            // If engine does not support groups, remove group information from the context info -
+            // use the old format instead (true = admin, array = user contexts).
+            if (!$this->engine->supports_group_filtering()) {
+                $contextinfo = $contextinfo->everything ? true : $contextinfo->usercontexts;
+            }
+
+            // Execute the actual query.
+            $docs = $this->engine->execute_query($formdata, $contextinfo, $limit);
         }
 
         return $docs;