diff --git a/admin/cli/install.php b/admin/cli/install.php index c2d08d129df9a..2ab997e445d57 100644 --- a/admin/cli/install.php +++ b/admin/cli/install.php @@ -394,8 +394,9 @@ cli_error(get_string('pathserrcreatedataroot', 'install', $a)); } } -$CFG->tempdir = $CFG->dataroot.'/temp'; -$CFG->cachedir = $CFG->dataroot.'/cache'; +$CFG->tempdir = $CFG->dataroot.'/temp'; +$CFG->cachedir = $CFG->dataroot.'/cache'; +$CFG->localcachedir = $CFG->dataroot.'/localcache'; // download required lang packs if ($CFG->lang !== 'en') { diff --git a/admin/tool/behat/cli/util.php b/admin/tool/behat/cli/util.php index cef4ea7e126ba..4ee20fe7b0148 100644 --- a/admin/tool/behat/cli/util.php +++ b/admin/tool/behat/cli/util.php @@ -151,6 +151,7 @@ // Unset cache and temp directories to reset them again with the new $CFG->dataroot. unset($CFG->cachedir); +unset($CFG->localcachedir); unset($CFG->tempdir); // Continues setup. diff --git a/config-dist.php b/config-dist.php index e22d246d37d78..22829ff85ba12 100644 --- a/config-dist.php +++ b/config-dist.php @@ -213,6 +213,7 @@ // $CFG->xsendfilealiases = array( // '/dataroot/' => $CFG->dataroot, // '/cachedir/' => '/var/www/moodle/cache', // for custom $CFG->cachedir locations +// '/localcachedir/' => '/var/local/cache', // for custom $CFG->localcachedir locations // '/tempdir/' => '/var/www/moodle/temp', // for custom $CFG->tempdir locations // '/filedir' => '/var/www/moodle/filedir', // for custom $CFG->filedir locations // ); @@ -353,10 +354,12 @@ // // It is possible to specify different cache and temp directories, use local fast filesystem // for normal web servers. Server clusters MUST use shared filesystem for cachedir! +// Localcachedir is intended for server clusters, it does not have to be shared by cluster nodes. // The directories must not be accessible via web. // -// $CFG->tempdir = '/var/www/moodle/temp'; -// $CFG->cachedir = '/var/www/moodle/cache'; +// $CFG->tempdir = '/var/www/moodle/temp'; // Files used during one HTTP request only. +// $CFG->cachedir = '/var/www/moodle/cache'; // Directory MUST BE SHARED by all cluster nodes, locking required. +// $CFG->localcachedir = '/var/local/cache'; // Intended for local node caching. // // Some filesystems such as NFS may not support file locking operations. // Locking resolves race conditions and is strongly recommended for production servers. diff --git a/install.php b/install.php index f591b00c52359..412952bef1dab 100644 --- a/install.php +++ b/install.php @@ -170,6 +170,7 @@ $CFG->dataroot = $config->dataroot; $CFG->tempdir = $CFG->dataroot.'/temp'; $CFG->cachedir = $CFG->dataroot.'/cache'; +$CFG->localcachedir = $CFG->dataroot.'/localcache'; $CFG->admin = $config->admin; $CFG->docroot = 'http://docs.moodle.org'; $CFG->langotherroot = $CFG->dataroot.'/lang'; diff --git a/lib/moodlelib.php b/lib/moodlelib.php index f79a2dbe7ca73..997048e6485e8 100644 --- a/lib/moodlelib.php +++ b/lib/moodlelib.php @@ -1560,6 +1560,12 @@ function purge_all_caches() { // hack: this script may get called after the purifier was initialised, // but we do not want to verify repeatedly this exists in each call make_cache_directory('htmlpurifier'); + + // This is the only place where we purge local caches, we are only adding files there. + // The $CFG->localcachedirpurged flag forces local directories to be purged on cluster nodes. + remove_dir($CFG->localcachedir, true); + set_config('localcachedirpurged', time()); + make_localcache_directory('', true); } /** diff --git a/lib/setup.php b/lib/setup.php index d0c2ab4abf029..56c22c5432101 100644 --- a/lib/setup.php +++ b/lib/setup.php @@ -37,7 +37,8 @@ * - $CFG->dirroot - Path to moodle's library folder on server's filesystem. * - $CFG->libdir - Path to moodle's library folder on server's filesystem. * - $CFG->tempdir - Path to moodle's temp file directory on server's filesystem. - * - $CFG->cachedir - Path to moodle's cache directory on server's filesystem. + * - $CFG->cachedir - Path to moodle's cache directory on server's filesystem (shared by cluster nodes). + * - $CFG->localcachedir - Path to moodle's local cache directory (not shared by cluster nodes). * * @global object $CFG * @name $CFG @@ -156,6 +157,11 @@ $CFG->cachedir = "$CFG->dataroot/cache"; } +// Allow overriding of localcachedir. +if (!isset($CFG->localcachedir)) { + $CFG->localcachedir = "$CFG->dataroot/localcache"; +} + // The current directory in PHP version 4.3.0 and above isn't necessarily the // directory of the script when run from the command line. The require_once() // would fail, so we'll have to chdir() diff --git a/lib/setuplib.php b/lib/setuplib.php index 4c9ba973708e8..81ebd50e20529 100644 --- a/lib/setuplib.php +++ b/lib/setuplib.php @@ -529,7 +529,7 @@ function get_exception_info($ex) { // Remove some absolute paths from message and debugging info. $searches = array(); $replaces = array(); - $cfgnames = array('tempdir', 'cachedir', 'themedir', + $cfgnames = array('tempdir', 'cachedir', 'localcachedir', 'themedir', 'langmenucachefile', 'langcacheroot', 'dataroot', 'dirroot'); foreach ($cfgnames as $cfgname) { if (property_exists($CFG, $cfgname)) { @@ -1311,7 +1311,10 @@ function make_upload_directory($directory, $exceptiononerror = true) { debugging('Use make_temp_directory() for creation of temporary directory and $CFG->tempdir to get the location.'); } else if (strpos($directory, 'cache/') === 0 or $directory === 'cache') { - debugging('Use make_cache_directory() for creation of chache directory and $CFG->cachedir to get the location.'); + debugging('Use make_cache_directory() for creation of cache directory and $CFG->cachedir to get the location.'); + + } else if (strpos($directory, 'localcache/') === 0 or $directory === 'localcache') { + debugging('Use make_localcache_directory() for creation of local cache directory and $CFG->localcachedir to get the location.'); } protect_directory($CFG->dataroot); @@ -1340,6 +1343,8 @@ function make_temp_directory($directory, $exceptiononerror = true) { /** * Create a directory under cachedir and make sure it is writable. * + * Note: this cache directory is shared by all cluster nodes. + * * @param string $directory the full path of the directory to be created under $CFG->cachedir * @param bool $exceptiononerror throw exception if error encountered * @return string|false Returns full path to directory if successful, false if not; may throw exception @@ -1355,6 +1360,58 @@ function make_cache_directory($directory, $exceptiononerror = true) { return make_writable_directory("$CFG->cachedir/$directory", $exceptiononerror); } +/** + * Create a directory under localcachedir and make sure it is writable. + * The files in this directory MUST NOT change, use revisions or content hashes to + * work around this limitation - this means you can only add new files here. + * + * The content of this directory gets purged automatically on all cluster nodes + * after calling purge_all_caches() before new data is written to this directory. + * + * Note: this local cache directory does not need to be shared by cluster nodes. + * + * @param string $directory the relative path of the directory to be created under $CFG->localcachedir + * @param bool $exceptiononerror throw exception if error encountered + * @return string|false Returns full path to directory if successful, false if not; may throw exception + */ +function make_localcache_directory($directory, $exceptiononerror = true) { + global $CFG; + + make_writable_directory($CFG->localcachedir, $exceptiononerror); + + if ($CFG->localcachedir !== "$CFG->dataroot/localcache") { + protect_directory($CFG->localcachedir); + } else { + protect_directory($CFG->dataroot); + } + + if (!isset($CFG->localcachedirpurged)) { + $CFG->localcachedirpurged = 0; + } + $timestampfile = "$CFG->localcachedir/.lastpurged"; + + if (!file_exists($timestampfile)) { + touch($timestampfile); + @chmod($timestampfile, $CFG->filepermissions); + + } else if (filemtime($timestampfile) < $CFG->localcachedirpurged) { + // This means our local cached dir was not purged yet. + remove_dir($CFG->localcachedir, true); + if ($CFG->localcachedir !== "$CFG->dataroot/localcache") { + protect_directory($CFG->localcachedir); + } + touch($timestampfile); + @chmod($timestampfile, $CFG->filepermissions); + clearstatcache(); + } + + if ($directory === '') { + return $CFG->localcachedir; + } + + return make_writable_directory("$CFG->localcachedir/$directory", $exceptiononerror); +} + /** * Checks if current user is a web crawler. * diff --git a/lib/testing/classes/util.php b/lib/testing/classes/util.php index 71ebacc7de797..4dfa12c4cb6f5 100644 --- a/lib/testing/classes/util.php +++ b/lib/testing/classes/util.php @@ -599,6 +599,7 @@ public static function reset_dataroot() { make_temp_directory(''); make_cache_directory(''); make_cache_directory('htmlpurifier'); + make_localcache_directory(''); // Reset the cache API so that it recreates it's required directories as well. cache_factory::reset(); // Purge all data from the caches. This is required for consistency. diff --git a/lib/tests/setuplib_test.php b/lib/tests/setuplib_test.php index 63431ee0efc2f..12dcb0e6784ae 100644 --- a/lib/tests/setuplib_test.php +++ b/lib/tests/setuplib_test.php @@ -31,7 +31,7 @@ * @copyright 2012 The Open University * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later */ -class core_setuplib_testcase extends basic_testcase { +class core_setuplib_testcase extends advanced_testcase { /** * Test get_docs_url_standard in the normal case when we should link to Moodle docs. @@ -126,7 +126,7 @@ public function test_exception_info_removes_serverpaths() { global $CFG; // This doesn't test them all possible ones, but these are set for unit tests. - $cfgnames = array('dataroot', 'dirroot', 'tempdir', 'cachedir'); + $cfgnames = array('dataroot', 'dirroot', 'tempdir', 'cachedir', 'localcachedir'); $fixture = ''; $expected = ''; @@ -142,4 +142,80 @@ public function test_exception_info_removes_serverpaths() { $this->assertContains($expected, $exceptioninfo->message, 'Exception message does not contain system paths'); $this->assertContains($expected, $exceptioninfo->debuginfo, 'Exception debug info does not contain system paths'); } + + public function test_localcachedir() { + global $CFG; + + $this->resetAfterTest(true); + + // Test default location - can not be modified in phpunit tests because we override everything in config.php. + $this->assertSame("$CFG->dataroot/localcache", $CFG->localcachedir); + + $now = time(); + $timestampfile = "$CFG->localcachedir/.lastpurged"; + + $dir = make_localcache_directory('', false); + $this->assertSame($CFG->localcachedir, $dir); + $this->assertFileNotExists("$CFG->localcachedir/.htaccess"); + $this->assertFileExists($timestampfile); + $this->assertGreaterThanOrEqual($now, filemtime($timestampfile)); + $this->assertLessThanOrEqual(time(), filemtime($timestampfile)); + + $dir = make_localcache_directory('test/test', false); + $this->assertSame("$CFG->localcachedir/test/test", $dir); + + // Test custom location. + $CFG->localcachedir = "$CFG->dataroot/testlocalcache"; + $now = time(); + $timestampfile = "$CFG->localcachedir/.lastpurged"; + $this->assertFileNotExists($timestampfile); + + $dir = make_localcache_directory('', false); + $this->assertSame($CFG->localcachedir, $dir); + $this->assertFileExists("$CFG->localcachedir/.htaccess"); + $this->assertFileExists($timestampfile); + $this->assertGreaterThanOrEqual($now, filemtime($timestampfile)); + $this->assertLessThanOrEqual(time(), filemtime($timestampfile)); + + $dir = make_localcache_directory('test', false); + $this->assertSame("$CFG->localcachedir/test", $dir); + + $prevtime = filemtime($timestampfile); + $dir = make_localcache_directory('pokus', false); + $this->assertSame("$CFG->localcachedir/pokus", $dir); + $this->assertSame($prevtime, filemtime($timestampfile)); + + + // Test purging. + $testfile = "$CFG->localcachedir/test/test.txt"; + $this->assertTrue(touch($testfile)); + + $now = time(); + set_config('localcachedirpurged', $now - 2); + purge_all_caches(); + $this->assertFileNotExists($testfile); + $this->assertFileNotExists(dirname($testfile)); + $this->assertFileExists($timestampfile); + $this->assertGreaterThanOrEqual($now, filemtime($timestampfile)); + $this->assertLessThanOrEqual(time(), filemtime($timestampfile)); + $this->assertGreaterThanOrEqual($now, $CFG->localcachedirpurged); + $this->assertLessThanOrEqual(time(), $CFG->localcachedirpurged); + + // Simulates purge_all_caches() on another server node. + make_localcache_directory('test', false); + $this->assertTrue(touch($testfile)); + set_config('localcachedirpurged', $now - 1); + $this->assertTrue(touch($timestampfile, $now - 2)); + clearstatcache(); + $this->assertSame($now - 2, filemtime($timestampfile)); + + $now = time(); + $dir = make_localcache_directory('', false); + $this->assertSame("$CFG->localcachedir", $dir); + $this->assertFileNotExists($testfile); + $this->assertFileNotExists(dirname($testfile)); + $this->assertFileExists($timestampfile); + $this->assertGreaterThanOrEqual($now, filemtime($timestampfile)); + $this->assertLessThanOrEqual(time(), filemtime($timestampfile)); + } } diff --git a/lib/upgrade.txt b/lib/upgrade.txt index 4c865c0c85304..65fa261ee16b7 100644 --- a/lib/upgrade.txt +++ b/lib/upgrade.txt @@ -10,6 +10,7 @@ information provided here is intended especially for developers. core_component::get_plugin_list() does not accept empty parameter any more. * Use core_text::* instead of textlib:: and also core_collator::* instead of collatorlib::*. * Use new function moodleform::mock_submit() to simulate form submission in unit tests (backported). +* New $CFG->localcachedir setting useful for cluster nodes. Admins have to update X-Sendfile aliases if used. DEPRECATIONS: Various previously deprecated functions have now been altered to throw DEBUG_DEVELOPER debugging notices diff --git a/lib/upgradelib.php b/lib/upgradelib.php index 32e4637f6ae24..1f552d5a57dde 100644 --- a/lib/upgradelib.php +++ b/lib/upgradelib.php @@ -1467,12 +1467,17 @@ function install_core($version, $verbose) { global $CFG, $DB; // We can not call purge_all_caches() yet, make sure the temp and cache dirs exist and are empty. - make_cache_directory('', true); remove_dir($CFG->cachedir.'', true); - make_temp_directory('', true); + make_cache_directory('', true); + + remove_dir($CFG->localcachedir.'', true); + make_localcache_directory('', true); + remove_dir($CFG->tempdir.'', true); - make_writable_directory($CFG->dataroot.'/muc', true); + make_temp_directory('', true); + remove_dir($CFG->dataroot.'/muc', true); + make_writable_directory($CFG->dataroot.'/muc', true); try { set_time_limit(600);