diff --git a/composer.json b/composer.json index 9c795b6c26bdf..92dfb41a8ecec 100644 --- a/composer.json +++ b/composer.json @@ -7,6 +7,7 @@ "require-dev": { "phpunit/phpunit": "5.5.*", "phpunit/dbUnit": "1.4.*", - "moodlehq/behat-extension": "3.33.1" + "moodlehq/behat-extension": "3.33.1", + "mikey179/vfsStream": "^1.6" } } diff --git a/composer.lock b/composer.lock index c757473c54df7..1fca2b9e93cc2 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#composer-lock-the-lock-file", "This file is @generated automatically" ], - "content-hash": "f4cfcd74744fbbced495458ea82fd314", + "content-hash": "751fc6623b264b33856167019b391053", "packages": [], "packages-dev": [ { @@ -836,6 +836,52 @@ ], "time": "2015-06-15T20:19:33+00:00" }, + { + "name": "mikey179/vfsStream", + "version": "v1.6.4", + "source": { + "type": "git", + "url": "https://github.com/mikey179/vfsStream.git", + "reference": "0247f57b2245e8ad2e689d7cee754b45fbabd592" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/mikey179/vfsStream/zipball/0247f57b2245e8ad2e689d7cee754b45fbabd592", + "reference": "0247f57b2245e8ad2e689d7cee754b45fbabd592", + "shasum": "" + }, + "require": { + "php": ">=5.3.0" + }, + "require-dev": { + "phpunit/phpunit": "~4.5" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "1.6.x-dev" + } + }, + "autoload": { + "psr-0": { + "org\\bovigo\\vfs\\": "src/main/php" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Frank Kleine", + "homepage": "http://frankkleine.de/", + "role": "Developer" + } + ], + "description": "Virtual file system to mock the real file system in unit tests.", + "homepage": "http://vfs.bovigo.org/", + "time": "2016-07-18T14:02:57+00:00" + }, { "name": "moodlehq/behat-extension", "version": "v3.33.1", @@ -887,16 +933,16 @@ }, { "name": "myclabs/deep-copy", - "version": "1.5.5", + "version": "1.6.0", "source": { "type": "git", "url": "https://github.com/myclabs/DeepCopy.git", - "reference": "399c1f9781e222f6eb6cc238796f5200d1b7f108" + "reference": "5a5a9fc8025a08d8919be87d6884d5a92520cefe" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/myclabs/DeepCopy/zipball/399c1f9781e222f6eb6cc238796f5200d1b7f108", - "reference": "399c1f9781e222f6eb6cc238796f5200d1b7f108", + "url": "https://api.github.com/repos/myclabs/DeepCopy/zipball/5a5a9fc8025a08d8919be87d6884d5a92520cefe", + "reference": "5a5a9fc8025a08d8919be87d6884d5a92520cefe", "shasum": "" }, "require": { @@ -925,7 +971,7 @@ "object", "object graph" ], - "time": "2016-10-31T17:19:45+00:00" + "time": "2017-01-26T22:05:40+00:00" }, { "name": "phpdocumentor/reflection-common", @@ -1197,16 +1243,16 @@ }, { "name": "phpunit/php-code-coverage", - "version": "4.0.4", + "version": "4.0.5", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/php-code-coverage.git", - "reference": "c14196e64a78570034afd0b7a9f3757ba71c2a0a" + "reference": "c19cfc7cbb0e9338d8c469c7eedecc2a428b0971" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/c14196e64a78570034afd0b7a9f3757ba71c2a0a", - "reference": "c14196e64a78570034afd0b7a9f3757ba71c2a0a", + "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/c19cfc7cbb0e9338d8c469c7eedecc2a428b0971", + "reference": "c19cfc7cbb0e9338d8c469c7eedecc2a428b0971", "shasum": "" }, "require": { @@ -1256,7 +1302,7 @@ "testing", "xunit" ], - "time": "2016-12-20T15:22:42+00:00" + "time": "2017-01-20T15:06:43+00:00" }, { "name": "phpunit/php-file-iterator", @@ -1725,16 +1771,16 @@ }, { "name": "sebastian/comparator", - "version": "1.2.2", + "version": "1.2.4", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/comparator.git", - "reference": "6a1ed12e8b2409076ab22e3897126211ff8b1f7f" + "reference": "2b7424b55f5047b47ac6e5ccb20b2aea4011d9be" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/comparator/zipball/6a1ed12e8b2409076ab22e3897126211ff8b1f7f", - "reference": "6a1ed12e8b2409076ab22e3897126211ff8b1f7f", + "url": "https://api.github.com/repos/sebastianbergmann/comparator/zipball/2b7424b55f5047b47ac6e5ccb20b2aea4011d9be", + "reference": "2b7424b55f5047b47ac6e5ccb20b2aea4011d9be", "shasum": "" }, "require": { @@ -1785,7 +1831,7 @@ "compare", "equality" ], - "time": "2016-11-19T09:18:40+00:00" + "time": "2017-01-29T09:50:25+00:00" }, { "name": "sebastian/diff", diff --git a/lib/filestorage/file_storage.php b/lib/filestorage/file_storage.php index c481ebe8b5bfb..3282796c65db9 100644 --- a/lib/filestorage/file_storage.php +++ b/lib/filestorage/file_storage.php @@ -43,16 +43,13 @@ * @since Moodle 2.0 */ class file_storage { - /** @var string Directory with file contents */ - private $filedir; - /** @var string Contents of deleted files not needed any more */ - private $trashdir; + /** @var string tempdir */ private $tempdir; - /** @var int Permissions for new directories */ - private $dirpermissions; - /** @var int Permissions for new files */ - private $filepermissions; + + /** @var file_system filesystem */ + private $filesystem; + /** @var array List of formats supported by unoconv */ private $unoconvformats; @@ -74,43 +71,46 @@ class file_storage { /** Any other error */ const UNOCONVPATH_ERROR = 'error'; - /** * Constructor - do not use directly use {@link get_file_storage()} call instead. - * - * @param string $filedir full path to pool directory - * @param string $trashdir temporary storage of deleted area - * @param string $tempdir temporary storage of various files - * @param int $dirpermissions new directory permissions - * @param int $filepermissions new file permissions */ - public function __construct($filedir, $trashdir, $tempdir, $dirpermissions, $filepermissions) { - global $CFG; + public function __construct() { + // The tempdir must always remain on disk, but shared between all ndoes in a cluster. Its content is not subject + // to the file_system abstraction. + $this->tempdir = make_temp_directory('filestorage'); - $this->filedir = $filedir; - $this->trashdir = $trashdir; - $this->tempdir = $tempdir; - $this->dirpermissions = $dirpermissions; - $this->filepermissions = $filepermissions; + $this->setup_file_system(); + } - // make sure the file pool directory exists - if (!is_dir($this->filedir)) { - if (!mkdir($this->filedir, $this->dirpermissions, true)) { - throw new file_exception('storedfilecannotcreatefiledirs'); // permission trouble - } - // place warning file in file pool root - if (!file_exists($this->filedir.'/warning.txt')) { - file_put_contents($this->filedir.'/warning.txt', - 'This directory contains the content of uploaded files and is controlled by Moodle code. Do not manually move, change or rename any of the files and subdirectories here.'); - chmod($this->filedir.'/warning.txt', $CFG->filepermissions); - } - } - // make sure the file pool directory exists - if (!is_dir($this->trashdir)) { - if (!mkdir($this->trashdir, $this->dirpermissions, true)) { - throw new file_exception('storedfilecannotcreatefiledirs'); // permission trouble + /** + * Complete setup procedure for the file_system component. + * + * @return file_system + */ + public function setup_file_system() { + global $CFG; + if ($this->filesystem === null) { + require_once($CFG->libdir . '/filestorage/file_system.php'); + if (!empty($CFG->alternative_file_system_class)) { + $class = $CFG->alternative_file_system_class; + } else { + // The default file_system is the filedir. + require_once($CFG->libdir . '/filestorage/file_system_filedir.php'); + $class = file_system_filedir::class; } + $this->filesystem = new $class(); } + + return $this->filesystem; + } + + /** + * Return the file system instance. + * + * @return file_system + */ + public function get_file_system() { + return $this->filesystem; } /** @@ -173,7 +173,7 @@ public function file_exists_by_hash($pathnamehash) { * @return stored_file instance of file abstraction class */ public function get_file_instance(stdClass $filerecord) { - $storedfile = new stored_file($this, $filerecord, $this->filedir); + $storedfile = new stored_file($this, $filerecord); return $storedfile; } @@ -1502,7 +1502,7 @@ public function create_file_from_pathname($filerecord, $pathname) { $newrecord->id = $DB->insert_record('files', $newrecord); } catch (dml_exception $e) { if ($newfile) { - $this->deleted_file_cleanup($newrecord->contenthash); + $this->move_to_trash($newrecord->contenthash); } throw new stored_file_creation_exception($newrecord->contextid, $newrecord->component, $newrecord->filearea, $newrecord->itemid, $newrecord->filepath, $newrecord->filename, $e->debuginfo); @@ -1609,9 +1609,11 @@ public function create_file_from_string($filerecord, $content) { $newrecord->sortorder = $filerecord->sortorder; list($newrecord->contenthash, $newrecord->filesize, $newfile) = $this->add_string_to_pool($content); - $filepathname = $this->path_from_hash($newrecord->contenthash) . '/' . $newrecord->contenthash; - // get mimetype by magic bytes - $newrecord->mimetype = empty($filerecord->mimetype) ? $this->mimetype($filepathname, $filerecord->filename) : $filerecord->mimetype; + if (empty($filerecord->mimetype)) { + $newrecord->mimetype = $this->filesystem->mimetype_from_hash($newrecord->contenthash, $newrecord->filename); + } else { + $newrecord->mimetype = $filerecord->mimetype; + } $newrecord->pathnamehash = $this->get_pathname_hash($newrecord->contextid, $newrecord->component, $newrecord->filearea, $newrecord->itemid, $newrecord->filepath, $newrecord->filename); @@ -1619,7 +1621,7 @@ public function create_file_from_string($filerecord, $content) { $newrecord->id = $DB->insert_record('files', $newrecord); } catch (dml_exception $e) { if ($newfile) { - $this->deleted_file_cleanup($newrecord->contenthash); + $this->move_to_trash($newrecord->contenthash); } throw new stored_file_creation_exception($newrecord->contextid, $newrecord->component, $newrecord->filearea, $newrecord->itemid, $newrecord->filepath, $newrecord->filename, $e->debuginfo); @@ -1723,16 +1725,19 @@ public function create_file_from_reference($filerecord, $repositoryid, $referenc throw new file_reference_exception($repositoryid, $reference, null, null, $e->getMessage()); } - if (isset($filerecord->contenthash) && $this->content_exists($filerecord->contenthash)) { - // there was specified the contenthash for a file already stored in moodle filepool + $existingfile = null; + if (isset($filerecord->contenthash)) { + $existingfile = $DB->get_record('files', array('contenthash' => $filerecord->contenthash)); + } + if (!empty($existingfile)) { + // There is an existing file already available. if (empty($filerecord->filesize)) { - $filepathname = $this->path_from_hash($filerecord->contenthash) . '/' . $filerecord->contenthash; - $filerecord->filesize = filesize($filepathname); + $filerecord->filesize = $existingfile->filesize; } else { $filerecord->filesize = clean_param($filerecord->filesize, PARAM_INT); } } else { - // atempt to get the result of last synchronisation for this reference + // Attempt to get the result of last synchronisation for this reference. $lastcontent = $DB->get_record('files', array('referencefileid' => $filerecord->referencefileid), 'id, contenthash, filesize', IGNORE_MULTIPLE); if ($lastcontent) { @@ -1751,7 +1756,7 @@ public function create_file_from_reference($filerecord, $repositoryid, $referenc $filerecord->id = $DB->insert_record('files', $filerecord); } catch (dml_exception $e) { if (!empty($newfile)) { - $this->deleted_file_cleanup($filerecord->contenthash); + $this->move_to_trash($filerecord->contenthash); } throw new stored_file_creation_exception($filerecord->contextid, $filerecord->component, $filerecord->filearea, $filerecord->itemid, $filerecord->filepath, $filerecord->filename, $e->debuginfo); @@ -1934,98 +1939,7 @@ public function convert_image($filerecord, $fid, $newwidth = null, $newheight = * @return array (contenthash, filesize, newfile) */ public function add_file_to_pool($pathname, $contenthash = NULL) { - global $CFG; - - if (!is_readable($pathname)) { - throw new file_exception('storedfilecannotread', '', $pathname); - } - - $filesize = filesize($pathname); - if ($filesize === false) { - throw new file_exception('storedfilecannotread', '', $pathname); - } - - if (is_null($contenthash)) { - $contenthash = sha1_file($pathname); - } else if ($CFG->debugdeveloper) { - $filehash = sha1_file($pathname); - if ($filehash === false) { - throw new file_exception('storedfilecannotread', '', $pathname); - } - if ($filehash !== $contenthash) { - // Hopefully this never happens, if yes we need to fix calling code. - debugging("Invalid contenthash submitted for file $pathname", DEBUG_DEVELOPER); - $contenthash = $filehash; - } - } - if ($contenthash === false) { - throw new file_exception('storedfilecannotread', '', $pathname); - } - - if ($filesize > 0 and $contenthash === sha1('')) { - // Did the file change or is sha1_file() borked for this file? - clearstatcache(); - $contenthash = sha1_file($pathname); - $filesize = filesize($pathname); - - if ($contenthash === false or $filesize === false) { - throw new file_exception('storedfilecannotread', '', $pathname); - } - if ($filesize > 0 and $contenthash === sha1('')) { - // This is very weird... - throw new file_exception('storedfilecannotread', '', $pathname); - } - } - - $hashpath = $this->path_from_hash($contenthash); - $hashfile = "$hashpath/$contenthash"; - - $newfile = true; - - if (file_exists($hashfile)) { - if (filesize($hashfile) === $filesize) { - return array($contenthash, $filesize, false); - } - if (sha1_file($hashfile) === $contenthash) { - // Jackpot! We have a sha1 collision. - mkdir("$this->filedir/jackpot/", $this->dirpermissions, true); - copy($pathname, "$this->filedir/jackpot/{$contenthash}_1"); - copy($hashfile, "$this->filedir/jackpot/{$contenthash}_2"); - throw new file_pool_content_exception($contenthash); - } - debugging("Replacing invalid content file $contenthash"); - unlink($hashfile); - $newfile = false; - } - - if (!is_dir($hashpath)) { - if (!mkdir($hashpath, $this->dirpermissions, true)) { - // Permission trouble. - throw new file_exception('storedfilecannotcreatefiledirs'); - } - } - - // Let's try to prevent some race conditions. - - $prev = ignore_user_abort(true); - @unlink($hashfile.'.tmp'); - if (!copy($pathname, $hashfile.'.tmp')) { - // Borked permissions or out of disk space. - ignore_user_abort($prev); - throw new file_exception('storedfilecannotcreatefile'); - } - if (filesize($hashfile.'.tmp') !== $filesize) { - // This should not happen. - unlink($hashfile.'.tmp'); - ignore_user_abort($prev); - throw new file_exception('storedfilecannotcreatefile'); - } - rename($hashfile.'.tmp', $hashfile); - chmod($hashfile, $this->filepermissions); // Fix permissions if needed. - @unlink($hashfile.'.tmp'); // Just in case anything fails in a weird way. - ignore_user_abort($prev); - - return array($contenthash, $filesize, $newfile); + return $this->filesystem->add_file_from_path($pathname, $contenthash); } /** @@ -2035,66 +1949,7 @@ public function add_file_to_pool($pathname, $contenthash = NULL) { * @return array (contenthash, filesize, newfile) */ public function add_string_to_pool($content) { - global $CFG; - - $contenthash = sha1($content); - $filesize = strlen($content); // binary length - - $hashpath = $this->path_from_hash($contenthash); - $hashfile = "$hashpath/$contenthash"; - - $newfile = true; - - if (file_exists($hashfile)) { - if (filesize($hashfile) === $filesize) { - return array($contenthash, $filesize, false); - } - if (sha1_file($hashfile) === $contenthash) { - // Jackpot! We have a sha1 collision. - mkdir("$this->filedir/jackpot/", $this->dirpermissions, true); - copy($hashfile, "$this->filedir/jackpot/{$contenthash}_1"); - file_put_contents("$this->filedir/jackpot/{$contenthash}_2", $content); - throw new file_pool_content_exception($contenthash); - } - debugging("Replacing invalid content file $contenthash"); - unlink($hashfile); - $newfile = false; - } - - if (!is_dir($hashpath)) { - if (!mkdir($hashpath, $this->dirpermissions, true)) { - // Permission trouble. - throw new file_exception('storedfilecannotcreatefiledirs'); - } - } - - // Hopefully this works around most potential race conditions. - - $prev = ignore_user_abort(true); - - if (!empty($CFG->preventfilelocking)) { - $newsize = file_put_contents($hashfile.'.tmp', $content); - } else { - $newsize = file_put_contents($hashfile.'.tmp', $content, LOCK_EX); - } - - if ($newsize === false) { - // Borked permissions most likely. - ignore_user_abort($prev); - throw new file_exception('storedfilecannotcreatefile'); - } - if (filesize($hashfile.'.tmp') !== $filesize) { - // Out of disk space? - unlink($hashfile.'.tmp'); - ignore_user_abort($prev); - throw new file_exception('storedfilecannotcreatefile'); - } - rename($hashfile.'.tmp', $hashfile); - chmod($hashfile, $this->filepermissions); // Fix permissions if needed. - @unlink($hashfile.'.tmp'); // Just in case anything fails in a weird way. - ignore_user_abort($prev); - - return array($contenthash, $filesize, $newfile); + return $this->filesystem->add_file_from_string($content); } /** @@ -2106,11 +1961,7 @@ public function add_string_to_pool($content) { * @return bool success */ public function xsendfile($contenthash) { - global $CFG; - require_once("$CFG->libdir/xsendfilelib.php"); - - $hashpath = $this->path_from_hash($contenthash); - return xsendfile("$hashpath/$contenthash"); + return $this->filesystem->xsendfile($contenthash); } /** @@ -2118,39 +1969,12 @@ public function xsendfile($contenthash) { * * @param string $contenthash * @return bool + * @deprecated since 3.3 */ public function content_exists($contenthash) { - $dir = $this->path_from_hash($contenthash); - $filepath = $dir . '/' . $contenthash; - return file_exists($filepath); - } + debugging('The content_exists function has been deprecated and should no longer be used.', DEBUG_DEVELOPER); - /** - * Return path to file with given hash. - * - * NOTE: must not be public, files in pool must not be modified - * - * @param string $contenthash content hash - * @return string expected file location - */ - protected function path_from_hash($contenthash) { - $l1 = $contenthash[0].$contenthash[1]; - $l2 = $contenthash[2].$contenthash[3]; - return "$this->filedir/$l1/$l2"; - } - - /** - * Return path to file with given hash. - * - * NOTE: must not be public, files in pool must not be modified - * - * @param string $contenthash content hash - * @return string expected file location - */ - protected function trash_path_from_hash($contenthash) { - $l1 = $contenthash[0].$contenthash[1]; - $l2 = $contenthash[2].$contenthash[3]; - return "$this->trashdir/$l1/$l2"; + return false; } /** @@ -2158,79 +1982,12 @@ protected function trash_path_from_hash($contenthash) { * * @param stored_file $file stored_file instance * @return bool success + * @deprecated since 3.3 */ public function try_content_recovery($file) { - $contenthash = $file->get_contenthash(); - $trashfile = $this->trash_path_from_hash($contenthash).'/'.$contenthash; - if (!is_readable($trashfile)) { - if (!is_readable($this->trashdir.'/'.$contenthash)) { - return false; - } - // nice, at least alternative trash file in trash root exists - $trashfile = $this->trashdir.'/'.$contenthash; - } - if (filesize($trashfile) != $file->get_filesize() or sha1_file($trashfile) != $contenthash) { - //weird, better fail early - return false; - } - $contentdir = $this->path_from_hash($contenthash); - $contentfile = $contentdir.'/'.$contenthash; - if (file_exists($contentfile)) { - //strange, no need to recover anything - return true; - } - if (!is_dir($contentdir)) { - if (!mkdir($contentdir, $this->dirpermissions, true)) { - return false; - } - } - return rename($trashfile, $contentfile); - } - - /** - * Marks pool file as candidate for deleting. - * - * DO NOT call directly - reserved for core!! - * - * @param string $contenthash - */ - public function deleted_file_cleanup($contenthash) { - global $DB; - - if ($contenthash === sha1('')) { - // No need to delete empty content file with sha1('') content hash. - return; - } + debugging('The try_content_recovery function has been deprecated and should no longer be used.', DEBUG_DEVELOPER); - //Note: this section is critical - in theory file could be reused at the same - // time, if this happens we can still recover the file from trash - if ($DB->record_exists('files', array('contenthash'=>$contenthash))) { - // file content is still used - return; - } - //move content file to trash - $contentfile = $this->path_from_hash($contenthash).'/'.$contenthash; - if (!file_exists($contentfile)) { - //weird, but no problem - return; - } - $trashpath = $this->trash_path_from_hash($contenthash); - $trashfile = $trashpath.'/'.$contenthash; - if (file_exists($trashfile)) { - // we already have this content in trash, no need to move it there - unlink($contentfile); - return; - } - if (!is_dir($trashpath)) { - mkdir($trashpath, $this->dirpermissions, true); - } - rename($contentfile, $trashfile); - - // Fix permissions, only if needed. - $currentperms = octdec(substr(decoct(fileperms($trashfile)), -4)); - if ((int)$this->filepermissions !== $currentperms) { - chmod($trashfile, $this->filepermissions); - } + return false; } /** @@ -2509,27 +2266,46 @@ public function import_external_file(stored_file $storedfile, $maxbytes = 0) { } /** - * Return mimetype by given file pathname + * Return mimetype by given file pathname. * * If file has a known extension, we return the mimetype based on extension. * Otherwise (when possible) we try to get the mimetype from file contents. * - * @param string $pathname full path to the file - * @param string $filename correct file name with extension, if omitted will be taken from $path + * @param string $fullpath Full path to the file on disk + * @param string $filename Correct file name with extension, if omitted will be taken from $path * @return string */ - public static function mimetype($pathname, $filename = null) { + public static function mimetype($fullpath, $filename = null) { if (empty($filename)) { - $filename = $pathname; + $filename = $fullpath; } + + // The mimeinfo function determines the mimetype purely based on the file extension. $type = mimeinfo('type', $filename); - if ($type === 'document/unknown' && class_exists('finfo') && file_exists($pathname)) { - $finfo = new finfo(FILEINFO_MIME_TYPE); - $type = mimeinfo_from_type('type', $finfo->file($pathname)); + + if ($type === 'document/unknown') { + // The type is unknown. Inspect the file now. + $type = self::mimetype_from_file($fullpath); } return $type; } + /** + * Inspect a file on disk for it's mimetype. + * + * @param string $fullpath Path to file on disk + * @return string The mimetype + */ + public static function mimetype_from_file($fullpath) { + if (file_exists($fullpath)) { + // The type is unknown. Attempt to look up the file type now. + $finfo = new finfo(FILEINFO_MIME_TYPE); + return mimeinfo_from_type('type', $finfo->file($fullpath)); + } + + return 'document/unknown'; + } + /** * Cron cleanup job. */ @@ -2614,10 +2390,9 @@ public function cron() { $rs->close(); mtrace('done.'); - mtrace('Deleting trash files... ', ''); + mtrace('Call filesystem cron tasks.', ''); cron_trace_time_and_memory(); - fulldelete($this->trashdir); - set_config('fileslastcleanup', time()); + $this->filesystem->cron(); mtrace('done.'); } } diff --git a/lib/filestorage/file_system.php b/lib/filestorage/file_system.php new file mode 100644 index 0000000000000..bfee09ca2260c --- /dev/null +++ b/lib/filestorage/file_system.php @@ -0,0 +1,561 @@ +. + +/** + * Core file system class definition. + * + * @package core_files + * @copyright 2017 Andrew Nicols + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +defined('MOODLE_INTERNAL') || die(); + +/** + * File system class used for low level access to real files in filedir. + * + * @package core_files + * @category files + * @copyright 2017 Andrew Nicols + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +abstract class file_system { + + /** + * Private clone method to prevent cloning of the instance. + */ + final protected function __clone() { + return; + } + + /** + * Private wakeup method to prevent unserialising of the instance. + */ + final protected function __wakeup() { + return; + } + + /** + * Output the content of the specified stored file. + * + * Note, this is different to get_content() as it uses the built-in php + * readfile function which is more efficient. + * + * @param stored_file $file The file to serve. + * @return void + */ + public function readfile(stored_file $file) { + if ($this->is_file_readable_locally_by_storedfile($file, false)) { + $path = $this->get_local_path_from_storedfile($file, false); + } else { + $path = $this->get_remote_path_from_storedfile($file); + } + readfile_allow_large($path, $file->get_filesize()); + } + + /** + * Get the full path on disk for the specified stored file. + * + * Note: This must return a consistent path for the file's contenthash + * and the path _will_ be in a standard local format. + * Streamable paths will not work. + * A local copy of the file _will_ be fetched if $fetchifnotfound is tree. + * + * The $fetchifnotfound allows you to determine the expected path of the file. + * + * @param stored_file $file The file to serve. + * @param bool $fetchifnotfound Whether to attempt to fetch from the remote path if not found. + * @return string full path to pool file with file content + */ + protected function get_local_path_from_storedfile(stored_file $file, $fetchifnotfound = false) { + return $this->get_local_path_from_hash($file->get_contenthash(), $fetchifnotfound); + } + + /** + * Get a remote filepath for the specified stored file. + * + * This is typically either the same as the local filepath, or it is a streamable resource. + * + * See https://secure.php.net/manual/en/wrappers.php for further information on valid wrappers. + * + * @param stored_file $file The file to serve. + * @return string full path to pool file with file content + */ + protected function get_remote_path_from_storedfile(stored_file $file) { + return $this->get_remote_path_from_hash($file->get_contenthash(), false); + } + + /** + * Get the full path for the specified hash, including the path to the filedir. + * + * Note: This must return a consistent path for the file's contenthash + * and the path _will_ be in a standard local format. + * Streamable paths will not work. + * A local copy of the file _will_ be fetched if $fetchifnotfound is tree. + * + * The $fetchifnotfound allows you to determine the expected path of the file. + * + * @param string $contenthash The content hash + * @param bool $fetchifnotfound Whether to attempt to fetch from the remote path if not found. + * @return string The full path to the content file + */ + abstract protected function get_local_path_from_hash($contenthash, $fetchifnotfound = false); + + /** + * Get the full path for the specified hash, including the path to the filedir. + * + * This is typically either the same as the local filepath, or it is a streamable resource. + * + * See https://secure.php.net/manual/en/wrappers.php for further information on valid wrappers. + * + * @param string $contenthash The content hash + * @return string The full path to the content file + */ + abstract protected function get_remote_path_from_hash($contenthash); + + /** + * Determine whether the file is present on the file system somewhere. + * A local copy of the file _will_ be fetched if $fetchifnotfound is tree. + * + * The $fetchifnotfound allows you to determine the expected path of the file. + * + * @param stored_file $file The file to ensure is available. + * @param bool $fetchifnotfound Whether to attempt to fetch from the remote path if not found. + * @return bool + */ + public function is_file_readable_locally_by_storedfile(stored_file $file, $fetchifnotfound = false) { + if (!$file->get_filesize()) { + // Files with empty size are either directories or empty. + // We handle these virtually. + return true; + } + + // Check to see if the file is currently readable. + $path = $this->get_local_path_from_storedfile($file, $fetchifnotfound); + if (is_readable($path)) { + return true; + } + + return false; + } + + /** + * Determine whether the file is present on the local file system somewhere. + * + * @param stored_file $file The file to ensure is available. + * @return bool + */ + public function is_file_readable_remotely_by_storedfile(stored_file $file) { + if (!$file->get_filesize()) { + // Files with empty size are either directories or empty. + // We handle these virtually. + return true; + } + + $path = $this->get_remote_path_from_storedfile($file, false); + if (is_readable($path)) { + return true; + } + + return false; + } + + /** + * Determine whether the file is present on the file system somewhere given + * the contenthash. + * + * @param string $contenthash The contenthash of the file to check. + * @param bool $fetchifnotfound Whether to attempt to fetch from the remote path if not found. + * @return bool + */ + public function is_file_readable_locally_by_hash($contenthash, $fetchifnotfound = false) { + if ($contenthash === sha1('')) { + // Files with empty size are either directories or empty. + // We handle these virtually. + return true; + } + + // This is called by file_storage::content_exists(), and in turn by the repository system. + $path = $this->get_local_path_from_hash($contenthash, $fetchifnotfound); + + // Note - it is not possible to perform a content recovery safely from a hash alone. + return is_readable($path); + } + + /** + * Determine whether the file is present locally on the file system somewhere given + * the contenthash. + * + * @param string $contenthash The contenthash of the file to check. + * @return bool + */ + public function is_file_readable_remotely_by_hash($contenthash) { + if ($contenthash === sha1('')) { + // Files with empty size are either directories or empty. + // We handle these virtually. + return true; + } + + $path = $this->get_remote_path_from_hash($contenthash, false); + + // Note - it is not possible to perform a content recovery safely from a hash alone. + return is_readable($path); + } + + /** + * Copy content of file to given pathname. + * + * @param stored_file $file The file to be copied + * @param string $target real path to the new file + * @return bool success + */ + abstract public function copy_content_from_storedfile(stored_file $file, $target); + + /** + * Remove the file with the specified contenthash. + * + * Note, if overriding this function, you _must_ check that the file is + * no longer in use - see {check_file_usage}. + * + * DO NOT call directly - reserved for core!! + * + * @param string $contenthash + */ + abstract public function remove_file($contenthash); + + /** + * Check whether a file is removable. + * + * This must be called prior to file removal. + * + * @param string $contenthash + * @return bool + */ + protected static function is_file_removable($contenthash) { + global $DB; + + if ($contenthash === sha1('')) { + // No need to delete empty content file with sha1('') content hash. + return false; + } + + // Note: This section is critical - in theory file could be reused at the same time, if this + // happens we can still recover the file from trash. + // Technically this is the responsibility of the file_storage API, but as this method is public, we go belt-and-braces. + if ($DB->record_exists('files', array('contenthash' => $contenthash))) { + // File content is still used. + return false; + } + + return true; + } + + /** + * Get the content of the specified stored file. + * + * Generally you will probably want to use readfile() to serve content, + * and where possible you should see if you can use + * get_content_file_handle and work with the file stream instead. + * + * @param stored_file $file The file to retrieve + * @return string The full file content + */ + public function get_content(stored_file $file) { + if (!$file->get_filesize()) { + // Directories are empty. Empty files are not worth fetching. + return ''; + } + + $source = $this->get_remote_path_from_storedfile($file); + return file_get_contents($source); + } + + /** + * List contents of archive. + * + * @param stored_file $file The archive to inspect + * @param file_packer $packer file packer instance + * @return array of file infos + */ + public function list_files($file, file_packer $packer) { + $archivefile = $this->get_local_path_from_storedfile($file, true); + return $packer->list_files($archivefile); + } + + /** + * Extract file to given file path (real OS filesystem), existing files are overwritten. + * + * @param stored_file $file The archive to inspect + * @param file_packer $packer File packer instance + * @param string $pathname Target directory + * @param file_progress $progress progress indicator callback or null if not required + * @return array|bool List of processed files; false if error + */ + public function extract_to_pathname(stored_file $file, file_packer $packer, $pathname, file_progress $progress = null) { + $archivefile = $this->get_local_path_from_storedfile($file, true); + return $packer->extract_to_pathname($archivefile, $pathname, null, $progress); + } + + /** + * Extract file to given file path (real OS filesystem), existing files are overwritten. + * + * @param stored_file $file The archive to inspect + * @param file_packer $packer file packer instance + * @param int $contextid context ID + * @param string $component component + * @param string $filearea file area + * @param int $itemid item ID + * @param string $pathbase path base + * @param int $userid user ID + * @param file_progress $progress Progress indicator callback or null if not required + * @return array|bool list of processed files; false if error + */ + public function extract_to_storage(stored_file $file, file_packer $packer, $contextid, + $component, $filearea, $itemid, $pathbase, $userid = null, file_progress $progress = null) { + + // Since we do not know which extractor we have, and whether it supports remote paths, use a local path here. + $archivefile = $this->get_local_path_from_storedfile($file, true); + return $packer->extract_to_storage($archivefile, $contextid, + $component, $filearea, $itemid, $pathbase, $userid, $progress); + } + + /** + * Add file/directory into archive. + * + * @param stored_file $file The file to archive + * @param file_archive $filearch file archive instance + * @param string $archivepath pathname in archive + * @return bool success + */ + public function add_storedfile_to_archive(stored_file $file, file_archive $filearch, $archivepath) { + if ($file->is_directory()) { + return $filearch->add_directory($archivepath); + } else { + // Since we do not know which extractor we have, and whether it supports remote paths, use a local path here. + return $filearch->add_file_from_pathname($archivepath, $this->get_local_path_from_storedfile($file, true)); + } + } + + /** + * Adds this file path to a curl request (POST only). + * + * @param stored_file $file The file to add to the curl request + * @param curl $curlrequest The curl request object + * @param string $key What key to use in the POST request + * @return void + * This needs the fullpath for the storedfile :/ + * Can this be achieved in some other fashion? + */ + public function add_to_curl_request(stored_file $file, &$curlrequest, $key) { + // Note: curl_file_create does not work with remote paths. + $path = $this->get_local_path_from_storedfile($file, true); + $curlrequest->_tmp_file_post_params[$key] = curl_file_create($path); + } + + /** + * Returns information about image. + * Information is determined from the file content + * + * @param stored_file $file The file to inspect + * @return mixed array with width, height and mimetype; false if not an image + */ + public function get_imageinfo(stored_file $file) { + if (!$this->is_image_from_storedfile($file)) { + return false; + } + + // Whilst get_imageinfo_from_path can use remote paths, it must download the entire file first. + // It is more efficient to use a local file when possible. + return $this->get_imageinfo_from_path($this->get_local_path_from_storedfile($file, true)); + } + + /** + * Attempt to determine whether the specified file is likely to be an + * image. + * Since this relies upon the mimetype stored in the files table, there + * may be times when this information is not 100% accurate. + * + * @param stored_file $file The file to check + * @return bool + */ + public function is_image_from_storedfile(stored_file $file) { + if (!$file->get_filesize()) { + // An empty file cannot be an image. + return false; + } + + $mimetype = $file->get_mimetype(); + if (!preg_match('|^image/|', $mimetype)) { + // The mimetype does not include image. + return false; + } + + // If it looks like an image, and it smells like an image, perhaps it's an image! + return true; + } + + /** + * Returns image information relating to the specified path or URL. + * + * @param string $path The path to pass to getimagesize. + * @return array Containing width, height, and mimetype. + */ + protected function get_imageinfo_from_path($path) { + $imageinfo = getimagesize($path); + + $image = array( + 'width' => $imageinfo[0], + 'height' => $imageinfo[1], + 'mimetype' => image_type_to_mime_type($imageinfo[2]), + ); + if (empty($image['width']) or empty($image['height']) or empty($image['mimetype'])) { + // GD can not parse it, sorry. + return false; + } + return $image; + } + + /** + * Serve file content using X-Sendfile header. + * Please make sure that all headers are already sent and the all + * access control checks passed. + * + * @param string $contenthash The content hash of the file to be served + * @return bool success + */ + public function xsendfile($contenthash) { + global $CFG; + require_once($CFG->libdir . "/xsendfilelib.php"); + + return xsendfile($this->get_remote_path_from_hash($contenthash)); + } + + /** + * Add the supplied file to the file system. + * + * Note: If overriding this function, it is advisable to store the file + * in the path returned by get_local_path_from_hash as there may be + * subsequent uses of the file in the same request. + * + * @param string $pathname Path to file currently on disk + * @param string $contenthash SHA1 hash of content if known (performance only) + * @return array (contenthash, filesize, newfile) + */ + abstract public function add_file_from_path($pathname, $contenthash = null); + + /** + * Add a file with the supplied content to the file system. + * + * Note: If overriding this function, it is advisable to store the file + * in the path returned by get_local_path_from_hash as there may be + * subsequent uses of the file in the same request. + * + * @param string $content file content - binary string + * @return array (contenthash, filesize, newfile) + */ + abstract public function add_file_from_string($content); + + /** + * Returns file handle - read only mode, no writing allowed into pool files! + * + * When you want to modify a file, create a new file and delete the old one. + * + * @param stored_file $file The file to retrieve a handle for + * @param int $type Type of file handle (FILE_HANDLE_xx constant) + * @return resource file handle + */ + public function get_content_file_handle(stored_file $file, $type = stored_file::FILE_HANDLE_FOPEN) { + $path = $this->get_remote_path_from_storedfile($file); + + return self::get_file_handle_for_path($path, $type); + } + + /** + * Return a file handle for the specified path. + * + * This abstraction should be used when overriding get_content_file_handle in a new file system. + * + * @param string $path The path to the file. This shoudl be any type of path that fopen and gzopen accept. + * @param int $type Type of file handle (FILE_HANDLE_xx constant) + * @return resource + * @throws coding_exception When an unexpected type of file handle is requested + */ + protected static function get_file_handle_for_path($path, $type = stored_file::FILE_HANDLE_FOPEN) { + switch ($type) { + case stored_file::FILE_HANDLE_FOPEN: + // Binary reading. + return fopen($path, 'rb'); + case stored_file::FILE_HANDLE_GZOPEN: + // Binary reading of file in gz format. + return gzopen($path, 'rb'); + default: + throw new coding_exception('Unexpected file handle type'); + } + } + + /** + * Retrieve the mime information for the specified stored file. + * + * @param string $contenthash + * @param string $filename + * @return string The MIME type. + */ + public function mimetype_from_hash($contenthash, $filename) { + $pathname = $this->get_remote_path_from_hash($contenthash); + $mimetype = file_storage::mimetype($pathname, $filename); + + if (!$this->is_file_readable_locally_by_hash($contenthash, false) && $mimetype === 'document/unknown') { + // The type is unknown, but the full checks weren't completed because the file isn't locally available. + // Ensure we have a local copy and try again. + $pathname = $this->get_local_path_from_hash($contenthash, true); + + $mimetype = file_storage::mimetype_from_file($pathname); + } + + return $mimetype; + } + + /** + * Retrieve the mime information for the specified stored file. + * + * @param stored_file $file The stored file to retrieve mime information for + * @return string The MIME type. + */ + public function mimetype_from_storedfile($file) { + if (!$file->get_filesize()) { + // Files with an empty filesize are treated as directories and have no mimetype. + return null; + } + $pathname = $this->get_remote_path_from_storedfile($file); + $mimetype = file_storage::mimetype($pathname, $file->get_filename()); + + if (!$this->is_file_readable_locally_by_storedfile($file) && $mimetype === 'document/unknown') { + // The type is unknown, but the full checks weren't completed because the file isn't locally available. + // Ensure we have a local copy and try again. + $pathname = $this->get_local_path_from_storedfile($file, true); + + $mimetype = file_storage::mimetype_from_file($pathname); + } + + return $mimetype; + } + + /** + * Run any periodic tasks which must be performed. + */ + public function cron() { + } +} diff --git a/lib/filestorage/file_system_filedir.php b/lib/filestorage/file_system_filedir.php new file mode 100644 index 0000000000000..b307a5dc758e9 --- /dev/null +++ b/lib/filestorage/file_system_filedir.php @@ -0,0 +1,515 @@ +. + +/** + * Core file system class definition. + * + * @package core_files + * @copyright 2017 Andrew Nicols + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +defined('MOODLE_INTERNAL') || die(); + +/** + * File system class used for low level access to real files in filedir. + * + * @package core_files + * @category files + * @copyright 2017 Andrew Nicols + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class file_system_filedir extends file_system { + + /** + * @var string The path to the local copy of the filedir. + */ + protected $filedir = null; + + /** + * @var string The path to the trashdir. + */ + protected $trashdir = null; + + /** + * @var string Default directory permissions for new dirs. + */ + protected $dirpermissions = null; + + /** + * @var string Default file permissions for new files. + */ + protected $filepermissions = null; + + + /** + * Perform any custom setup for this type of file_system. + */ + public function __construct() { + global $CFG; + + if (isset($CFG->filedir)) { + $this->filedir = $CFG->filedir; + } else { + $this->filedir = $CFG->dataroot.'/filedir'; + } + + if (isset($CFG->trashdir)) { + $this->trashdir = $CFG->trashdir; + } else { + $this->trashdir = $CFG->dataroot.'/trashdir'; + } + + $this->dirpermissions = $CFG->directorypermissions; + $this->filepermissions = $CFG->filepermissions; + + // Make sure the file pool directory exists. + if (!is_dir($this->filedir)) { + if (!mkdir($this->filedir, $this->dirpermissions, true)) { + // Permission trouble. + throw new file_exception('storedfilecannotcreatefiledirs'); + } + + // Place warning file in file pool root. + if (!file_exists($this->filedir.'/warning.txt')) { + file_put_contents($this->filedir.'/warning.txt', + 'This directory contains the content of uploaded files and is controlled by Moodle code. ' . + 'Do not manually move, change or rename any of the files and subdirectories here.'); + chmod($this->filedir . '/warning.txt', $this->filepermissions); + } + } + + // Make sure the trashdir directory exists too. + if (!is_dir($this->trashdir)) { + if (!mkdir($this->trashdir, $this->dirpermissions, true)) { + // Permission trouble. + throw new file_exception('storedfilecannotcreatefiledirs'); + } + } + } + + /** + * Get the full path for the specified hash, including the path to the filedir. + * + * @param string $contenthash The content hash + * @param bool $fetchifnotfound Whether to attempt to fetch from the remote path if not found. + * @return string The full path to the content file + */ + protected function get_local_path_from_hash($contenthash, $fetchifnotfound = false) { + return $this->get_fulldir_from_hash($contenthash) . DIRECTORY_SEPARATOR . $contenthash; + } + + /** + * Get a remote filepath for the specified stored file. + * + * @param stored_file $file The file to fetch the path for + * @param bool $fetchifnotfound Whether to attempt to fetch from the remote path if not found. + * @return string The full path to the content file + */ + protected function get_local_path_from_storedfile(stored_file $file, $fetchifnotfound = false) { + $filepath = $this->get_local_path_from_hash($file->get_contenthash(), $fetchifnotfound); + + // Try content recovery. + if ($fetchifnotfound && !is_readable($filepath)) { + $this->recover_file($file); + } + + return $filepath; + } + + /** + * Get a remote filepath for the specified stored file. + * + * @param stored_file $file The file to serve. + * @return string full path to pool file with file content + */ + protected function get_remote_path_from_storedfile(stored_file $file) { + return $this->get_local_path_from_storedfile($file, false); + } + + /** + * Get the full path for the specified hash, including the path to the filedir. + * + * @param string $contenthash The content hash + * @return string The full path to the content file + */ + protected function get_remote_path_from_hash($contenthash) { + return $this->get_local_path_from_hash($contenthash, false); + } + + /** + * Get the full directory to the stored file, including the path to the + * filedir, and the directory which the file is actually in. + * + * Note: This function does not ensure that the file is present on disk. + * + * @param stored_file $file The file to fetch details for. + * @return string The full path to the content directory + */ + protected function get_fulldir_from_storedfile(stored_file $file) { + return $this->get_fulldir_from_hash($file->get_contenthash()); + } + + /** + * Get the full directory to the stored file, including the path to the + * filedir, and the directory which the file is actually in. + * + * @param string $contenthash The content hash + * @return string The full path to the content directory + */ + protected function get_fulldir_from_hash($contenthash) { + return $this->filedir . DIRECTORY_SEPARATOR . $this->get_contentdir_from_hash($contenthash); + } + + /** + * Get the content directory for the specified content hash. + * This is the directory that the file will be in, but without the + * fulldir. + * + * @param string $contenthash The content hash + * @return string The directory within filedir + */ + protected function get_contentdir_from_hash($contenthash) { + $l1 = $contenthash[0] . $contenthash[1]; + $l2 = $contenthash[2] . $contenthash[3]; + return "$l1/$l2"; + } + + /** + * Get the content path for the specified content hash within filedir. + * + * This does not include the filedir, and is often used by file systems + * as the object key for storage and retrieval. + * + * @param string $contenthash The content hash + * @return string The filepath within filedir + */ + protected function get_contentpath_from_hash($contenthash) { + return $this->get_contentdir_from_hash($contenthash) . "/$contenthash"; + } + + /** + * Get the full directory for the specified hash in the trash, including the path to the + * trashdir, and the directory which the file is actually in. + * + * @param string $contenthash The content hash + * @return string The full path to the trash directory + */ + protected function get_trash_fulldir_from_hash($contenthash) { + return $this->trashdir . DIRECTORY_SEPARATOR . $this->get_contentdir_from_hash($contenthash); + } + + /** + * Get the full path for the specified hash in the trash, including the path to the trashdir. + * + * @param string $contenthash The content hash + * @return string The full path to the trash file + */ + protected function get_trash_fullpath_from_hash($contenthash) { + return $this->trashdir . DIRECTORY_SEPARATOR . $this->get_contentpath_from_hash($contenthash); + } + + /** + * Copy content of file to given pathname. + * + * @param stored_file $file The file to be copied + * @param string $target real path to the new file + * @return bool success + */ + public function copy_content_from_storedfile(stored_file $file, $target) { + $source = $this->get_local_path_from_storedfile($file, true); + return copy($source, $target); + } + + /** + * Tries to recover missing content of file from trash. + * + * @param stored_file $file stored_file instance + * @return bool success + */ + protected function recover_file(stored_file $file) { + $contentfile = $this->get_local_path_from_storedfile($file, false); + + if (file_exists($contentfile)) { + // The file already exists on the file system. No need to recover. + return true; + } + + $contenthash = $file->get_contenthash(); + $contentdir = $this->get_fulldir_from_storedfile($file); + $trashfile = $this->get_trash_fullpath_from_hash($contenthash); + $alttrashfile = $this->trashdir . DIRECTORY_SEPARATOR . $contenthash; + + if (!is_readable($trashfile)) { + // The trash file was not found. Check the alternative trash file too just in case. + if (!is_readable($alttrashfile)) { + return false; + } + // The alternative trash file in trash root exists. + $trashfile = $alttrashfile; + } + + if (filesize($trashfile) != $file->get_filesize() or sha1_file($trashfile) != $contenthash) { + // The files are different. Leave this one in trash - something seems to be wrong with it. + return false; + } + + if (!is_dir($contentdir)) { + if (!mkdir($contentdir, $this->dirpermissions, true)) { + // Unable to create the target directory. + return false; + } + } + + // Perform a rename - these are generally atomic which gives us big + // performance wins, especially for large files. + return rename($trashfile, $contentfile); + } + + /** + * Marks pool file as candidate for deleting. + * + * @param string $contenthash + */ + public function remove_file($contenthash) { + if (!self::is_file_removable($contenthash)) { + // Don't remove the file - it's still in use. + return; + } + + if (!$this->is_file_readable_remotely_by_hash($contenthash)) { + // The file wasn't found in the first place. Just ignore it. + return; + } + + $trashpath = $this->get_trash_fulldir_from_hash($contenthash); + $trashfile = $this->get_trash_fullpath_from_hash($contenthash); + $contentfile = $this->get_local_path_from_hash($contenthash, true); + + if (!is_dir($trashpath)) { + mkdir($trashpath, $this->dirpermissions, true); + } + + if (file_exists($trashfile)) { + // A copy of this file is already in the trash. + // Remove the old version. + unlink($contentfile); + return; + } + + // Move the contentfile to the trash, and fix permissions as required. + rename($contentfile, $trashfile); + + // Fix permissions, only if needed. + $currentperms = octdec(substr(decoct(fileperms($trashfile)), -4)); + if ((int)$this->filepermissions !== $currentperms) { + chmod($trashfile, $this->filepermissions); + } + } + + /** + * Cleanup the trash directory. + */ + public function cron() { + $this->empty_trash(); + } + + protected function empty_trash() { + fulldelete($this->trashdir); + set_config('fileslastcleanup', time()); + } + + /** + * Add the supplied file to the file system. + * + * Note: If overriding this function, it is advisable to store the file + * in the path returned by get_local_path_from_hash as there may be + * subsequent uses of the file in the same request. + * + * @param string $pathname Path to file currently on disk + * @param string $contenthash SHA1 hash of content if known (performance only) + * @return array (contenthash, filesize, newfile) + */ + public function add_file_from_path($pathname, $contenthash = null) { + global $CFG; + + if (!is_readable($pathname)) { + throw new file_exception('storedfilecannotread', '', $pathname); + } + + $filesize = filesize($pathname); + if ($filesize === false) { + throw new file_exception('storedfilecannotread', '', $pathname); + } + + if (is_null($contenthash)) { + $contenthash = sha1_file($pathname); + } else if ($CFG->debugdeveloper) { + $filehash = sha1_file($pathname); + if ($filehash === false) { + throw new file_exception('storedfilecannotread', '', $pathname); + } + if ($filehash !== $contenthash) { + // Hopefully this never happens, if yes we need to fix calling code. + debugging("Invalid contenthash submitted for file $pathname", DEBUG_DEVELOPER); + $contenthash = $filehash; + } + } + if ($contenthash === false) { + throw new file_exception('storedfilecannotread', '', $pathname); + } + + if ($filesize > 0 and $contenthash === sha1('')) { + // Did the file change or is sha1_file() borked for this file? + clearstatcache(); + $contenthash = sha1_file($pathname); + $filesize = filesize($pathname); + + if ($contenthash === false or $filesize === false) { + throw new file_exception('storedfilecannotread', '', $pathname); + } + if ($filesize > 0 and $contenthash === sha1('')) { + // This is very weird... + throw new file_exception('storedfilecannotread', '', $pathname); + } + } + + $hashpath = $this->get_fulldir_from_hash($contenthash); + $hashfile = $this->get_local_path_from_hash($contenthash, false); + + $newfile = true; + + if (file_exists($hashfile)) { + if (filesize($hashfile) === $filesize) { + return array($contenthash, $filesize, false); + } + if (sha1_file($hashfile) === $contenthash) { + // Jackpot! We have a sha1 collision. + mkdir("$this->filedir/jackpot/", $this->dirpermissions, true); + copy($pathname, "$this->filedir/jackpot/{$contenthash}_1"); + copy($hashfile, "$this->filedir/jackpot/{$contenthash}_2"); + throw new file_pool_content_exception($contenthash); + } + debugging("Replacing invalid content file $contenthash"); + unlink($hashfile); + $newfile = false; + } + + if (!is_dir($hashpath)) { + if (!mkdir($hashpath, $this->dirpermissions, true)) { + // Permission trouble. + throw new file_exception('storedfilecannotcreatefiledirs'); + } + } + + // Let's try to prevent some race conditions. + + $prev = ignore_user_abort(true); + @unlink($hashfile.'.tmp'); + if (!copy($pathname, $hashfile.'.tmp')) { + // Borked permissions or out of disk space. + ignore_user_abort($prev); + throw new file_exception('storedfilecannotcreatefile'); + } + if (filesize($hashfile.'.tmp') !== $filesize) { + // This should not happen. + unlink($hashfile.'.tmp'); + ignore_user_abort($prev); + throw new file_exception('storedfilecannotcreatefile'); + } + rename($hashfile.'.tmp', $hashfile); + chmod($hashfile, $this->filepermissions); // Fix permissions if needed. + @unlink($hashfile.'.tmp'); // Just in case anything fails in a weird way. + ignore_user_abort($prev); + + return array($contenthash, $filesize, $newfile); + } + + /** + * Add a file with the supplied content to the file system. + * + * Note: If overriding this function, it is advisable to store the file + * in the path returned by get_local_path_from_hash as there may be + * subsequent uses of the file in the same request. + * + * @param string $content file content - binary string + * @return array (contenthash, filesize, newfile) + */ + public function add_file_from_string($content) { + global $CFG; + + $contenthash = sha1($content); + // Binary length. + $filesize = strlen($content); + + $hashpath = $this->get_fulldir_from_hash($contenthash); + $hashfile = $this->get_local_path_from_hash($contenthash, false); + + $newfile = true; + + if (file_exists($hashfile)) { + if (filesize($hashfile) === $filesize) { + return array($contenthash, $filesize, false); + } + if (sha1_file($hashfile) === $contenthash) { + // Jackpot! We have a sha1 collision. + mkdir("$this->filedir/jackpot/", $this->dirpermissions, true); + copy($hashfile, "$this->filedir/jackpot/{$contenthash}_1"); + file_put_contents("$this->filedir/jackpot/{$contenthash}_2", $content); + throw new file_pool_content_exception($contenthash); + } + debugging("Replacing invalid content file $contenthash"); + unlink($hashfile); + $newfile = false; + } + + if (!is_dir($hashpath)) { + if (!mkdir($hashpath, $this->dirpermissions, true)) { + // Permission trouble. + throw new file_exception('storedfilecannotcreatefiledirs'); + } + } + + // Hopefully this works around most potential race conditions. + + $prev = ignore_user_abort(true); + + if (!empty($CFG->preventfilelocking)) { + $newsize = file_put_contents($hashfile.'.tmp', $content); + } else { + $newsize = file_put_contents($hashfile.'.tmp', $content, LOCK_EX); + } + + if ($newsize === false) { + // Borked permissions most likely. + ignore_user_abort($prev); + throw new file_exception('storedfilecannotcreatefile'); + } + if (filesize($hashfile.'.tmp') !== $filesize) { + // Out of disk space? + unlink($hashfile.'.tmp'); + ignore_user_abort($prev); + throw new file_exception('storedfilecannotcreatefile'); + } + rename($hashfile.'.tmp', $hashfile); + chmod($hashfile, $this->filepermissions); // Fix permissions if needed. + @unlink($hashfile.'.tmp'); // Just in case anything fails in a weird way. + ignore_user_abort($prev); + + return array($contenthash, $filesize, $newfile); + } + +} diff --git a/lib/filestorage/stored_file.php b/lib/filestorage/stored_file.php index 7d3b64f86fdcb..192e7b8daf285 100644 --- a/lib/filestorage/stored_file.php +++ b/lib/filestorage/stored_file.php @@ -26,6 +26,7 @@ defined('MOODLE_INTERNAL') || die(); require_once($CFG->dirroot . '/lib/filestorage/file_progress.php'); +require_once($CFG->dirroot . '/lib/filestorage/file_system.php'); /** * Class representing local files stored in a sha1 file pool. @@ -44,10 +45,10 @@ class stored_file { private $fs; /** @var stdClass record from the files table left join files_reference table */ private $file_record; - /** @var string location of content files */ - private $filedir; /** @var repository repository plugin instance */ private $repository; + /** @var file_system filesystem instance */ + private $filesystem; /** * @var int Indicates a file handle of the type returned by fopen. @@ -65,13 +66,12 @@ class stored_file { * * @param file_storage $fs file storage instance * @param stdClass $file_record description of file - * @param string $filedir location of file directory with sh1 named content files + * @param string $deprecated */ - public function __construct(file_storage $fs, stdClass $file_record, $filedir) { + public function __construct(file_storage $fs, stdClass $file_record, $deprecated = null) { global $DB, $CFG; $this->fs = $fs; $this->file_record = clone($file_record); // prevent modifications - $this->filedir = $filedir; // keep secret, do not expose! if (!empty($file_record->repositoryid)) { require_once("$CFG->dirroot/repository/lib.php"); @@ -89,6 +89,8 @@ public function __construct(file_storage $fs, stdClass $file_record, $filedir) { $this->file_record->$key = null; } } + + $this->filesystem = $fs->get_file_system(); } /** @@ -179,15 +181,7 @@ protected function update($dataobject) { } } // Validate mimetype field - // we don't use {@link stored_file::get_content_file_location()} here becaues it will try to update file_record - $pathname = $this->get_pathname_by_contenthash(); - // try to recover the content from trash - if (!is_readable($pathname)) { - if (!$this->fs->try_content_recovery($this) or !is_readable($pathname)) { - throw new file_exception('storedfilecannotread', '', $pathname); - } - } - $mimetype = $this->fs->mimetype($pathname, $this->file_record->filename); + $mimetype = $this->filesystem->mimetype_from_storedfile($this); $this->file_record->mimetype = $mimetype; $DB->update_record('files', $this->file_record); @@ -255,8 +249,8 @@ public function replace_file_with(stored_file $newfile) { } $filerecord = new stdClass; - $contenthash = $newfile->get_contenthash(); - if ($this->fs->content_exists($contenthash)) { + if ($this->filesystem->is_file_readable_remotely_by_storedfile($newfile)) { + $contenthash = $newfile->get_contenthash(); $filerecord->contenthash = $contenthash; } else { throw new file_exception('storedfileproblem', 'Invalid contenthash, content must be already in filepool', $contenthash); @@ -357,40 +351,10 @@ public function delete() { } // Move pool file to trash if content not needed any more. - $this->fs->deleted_file_cleanup($this->file_record->contenthash); + $this->filesystem->remove_file($this->file_record->contenthash); return true; // BC only } - /** - * Get file pathname by contenthash - * - * NOTE, this function is not calling sync_external_file, it assume the contenthash is current - * Protected - developers must not gain direct access to this function. - * - * @return string full path to pool file with file content - */ - protected function get_pathname_by_contenthash() { - // Detect is local file or not. - $contenthash = $this->file_record->contenthash; - $l1 = $contenthash[0].$contenthash[1]; - $l2 = $contenthash[2].$contenthash[3]; - return "$this->filedir/$l1/$l2/$contenthash"; - } - - /** - * Get file pathname by given contenthash, this method will try to sync files - * - * Protected - developers must not gain direct access to this function. - * - * NOTE: do not make this public, we must not modify or delete the pool files directly! ;-) - * - * @return string full path to pool file with file content - **/ - protected function get_content_file_location() { - $this->sync_external_file(); - return $this->get_pathname_by_contenthash(); - } - /** * adds this file path to a curl request (POST only) * @@ -399,13 +363,7 @@ protected function get_content_file_location() { * @return void */ public function add_to_curl_request(&$curlrequest, $key) { - if (function_exists('curl_file_create')) { - // As of PHP 5.5, the usage of the @filename API for file uploading is deprecated. - $value = curl_file_create($this->get_content_file_location()); - } else { - $value = '@' . $this->get_content_file_location(); - } - $curlrequest->_tmp_file_post_params[$key] = $value; + return $this->filesystem->add_to_curl_request($this, $curlrequest, $key); } /** @@ -417,35 +375,14 @@ public function add_to_curl_request(&$curlrequest, $key) { * @return resource file handle */ public function get_content_file_handle($type = self::FILE_HANDLE_FOPEN) { - $path = $this->get_content_file_location(); - if (!is_readable($path)) { - if (!$this->fs->try_content_recovery($this) or !is_readable($path)) { - throw new file_exception('storedfilecannotread', '', $path); - } - } - switch ($type) { - case self::FILE_HANDLE_FOPEN: - // Binary reading. - return fopen($path, 'rb'); - case self::FILE_HANDLE_GZOPEN: - // Binary reading of file in gz format. - return gzopen($path, 'rb'); - default: - throw new coding_exception('Unexpected file handle type'); - } + return $this->filesystem->get_content_file_handle($this, $type); } /** * Dumps file content to page. */ public function readfile() { - $path = $this->get_content_file_location(); - if (!is_readable($path)) { - if (!$this->fs->try_content_recovery($this) or !is_readable($path)) { - throw new file_exception('storedfilecannotread', '', $path); - } - } - readfile_allow_large($path, $this->get_filesize()); + return $this->filesystem->readfile($this); } /** @@ -454,13 +391,7 @@ public function readfile() { * @return string content */ public function get_content() { - $path = $this->get_content_file_location(); - if (!is_readable($path)) { - if (!$this->fs->try_content_recovery($this) or !is_readable($path)) { - throw new file_exception('storedfilecannotread', '', $path); - } - } - return file_get_contents($this->get_content_file_location()); + return $this->filesystem->get_content($this); } /** @@ -470,13 +401,7 @@ public function get_content() { * @return bool success */ public function copy_content_to($pathname) { - $path = $this->get_content_file_location(); - if (!is_readable($path)) { - if (!$this->fs->try_content_recovery($this) or !is_readable($path)) { - throw new file_exception('storedfilecannotread', '', $path); - } - } - return copy($path, $pathname); + return $this->filesystem->copy_content_from_storedfile($this, $pathname); } /** @@ -509,8 +434,7 @@ public function copy_content_to_temp($dir = 'files', $fileprefix = 'tempup_') { * @return array of file infos */ public function list_files(file_packer $packer) { - $archivefile = $this->get_content_file_location(); - return $packer->list_files($archivefile); + return $this->filesystem->list_files($this, $packer); } /** @@ -523,8 +447,7 @@ public function list_files(file_packer $packer) { */ public function extract_to_pathname(file_packer $packer, $pathname, file_progress $progress = null) { - $archivefile = $this->get_content_file_location(); - return $packer->extract_to_pathname($archivefile, $pathname, null, $progress); + return $this->filesystem->extract_to_pathname($this, $packer, $pathname, $progress); } /** @@ -542,9 +465,9 @@ public function extract_to_pathname(file_packer $packer, $pathname, */ public function extract_to_storage(file_packer $packer, $contextid, $component, $filearea, $itemid, $pathbase, $userid = null, file_progress $progress = null) { - $archivefile = $this->get_content_file_location(); - return $packer->extract_to_storage($archivefile, $contextid, - $component, $filearea, $itemid, $pathbase, $userid, $progress); + + return $this->filesystem->extract_to_storage($this, $packer, $contextid, $component, $filearea, + $itemid, $pathbase, $userid, $progress); } /** @@ -555,15 +478,7 @@ public function extract_to_storage(file_packer $packer, $contextid, * @return bool success */ public function archive_file(file_archive $filearch, $archivepath) { - if ($this->is_directory()) { - return $filearch->add_directory($archivepath); - } else { - $path = $this->get_content_file_location(); - if (!is_readable($path)) { - return false; - } - return $filearch->add_file_from_pathname($archivepath, $path); - } + return $this->filesystem->add_storedfile_to_archive($this, $filearch, $archivepath); } /** @@ -573,22 +488,7 @@ public function archive_file(file_archive $filearch, $archivepath) { * @return mixed array with width, height and mimetype; false if not an image */ public function get_imageinfo() { - $path = $this->get_content_file_location(); - if (!is_readable($path)) { - if (!$this->fs->try_content_recovery($this) or !is_readable($path)) { - throw new file_exception('storedfilecannotread', '', $path); - } - } - $mimetype = $this->get_mimetype(); - if (!preg_match('|^image/|', $mimetype) || !filesize($path) || !($imageinfo = getimagesize($path))) { - return false; - } - $image = array('width'=>$imageinfo[0], 'height'=>$imageinfo[1], 'mimetype'=>image_type_to_mime_type($imageinfo[2])); - if (empty($image['width']) or empty($image['height']) or empty($image['mimetype'])) { - // gd can not parse it, sorry - return false; - } - return $image; + return $this->filesystem->get_imageinfo($this); } /** @@ -985,7 +885,7 @@ public function set_synchronized($contenthash, $filesize, $status = 0, $timemodi $this->file_record->timemodified = $timemodified; } if (isset($oldcontenthash)) { - $this->fs->deleted_file_cleanup($oldcontenthash); + $this->filesystem->remove_file($oldcontenthash); } } @@ -1051,14 +951,16 @@ public function generate_image_thumbnail($width, $height) { return false; } + $content = $this->get_content(); + // Fetch the image information for this image. - $imageinfo = @getimagesizefromstring($this->get_content()); + $imageinfo = @getimagesizefromstring($content); if (empty($imageinfo)) { return false; } // Create a new image from the file. - $original = @imagecreatefromstring($this->get_content()); + $original = @imagecreatefromstring($content); // Generate the thumbnail. return generate_image_thumbnail_from_image($original, $imageinfo, $width, $height); @@ -1075,14 +977,16 @@ public function resize_image($width, $height) { global $CFG; require_once($CFG->libdir . '/gdlib.php'); + $content = $this->get_content(); + // Fetch the image information for this image. - $imageinfo = @getimagesizefromstring($this->get_content()); + $imageinfo = @getimagesizefromstring($content); if (empty($imageinfo)) { return false; } // Create a new image from the file. - $original = @imagecreatefromstring($this->get_content()); + $original = @imagecreatefromstring($content); // Generate the resized image. return resize_image_from_image($original, $imageinfo, $width, $height); diff --git a/lib/filestorage/tests/file_storage_test.php b/lib/filestorage/tests/file_storage_test.php index f16ada741005f..46e97fc095851 100644 --- a/lib/filestorage/tests/file_storage_test.php +++ b/lib/filestorage/tests/file_storage_test.php @@ -64,7 +64,10 @@ public function test_create_file_from_string() { $this->assertTrue($DB->record_exists('files', array('pathnamehash'=>$pathhash))); - $location = test_stored_file_inspection::get_pretected_pathname($file); + $method = new ReflectionMethod('file_system', 'get_local_path_from_storedfile'); + $method->setAccessible(true); + $filesystem = $fs->get_file_system(); + $location = $method->invokeArgs($filesystem, array($file, true)); $this->assertFileExists($location); @@ -133,7 +136,10 @@ public function test_create_file_from_pathname() { $this->assertTrue($DB->record_exists('files', array('pathnamehash'=>$pathhash))); - $location = test_stored_file_inspection::get_pretected_pathname($file); + $method = new ReflectionMethod('file_system', 'get_local_path_from_storedfile'); + $method->setAccessible(true); + $filesystem = $fs->get_file_system(); + $location = $method->invokeArgs($filesystem, array($file, true)); $this->assertFileExists($location); @@ -1842,6 +1848,53 @@ public function test_get_unused_filename() { $this->expectException('coding_exception'); $fs->get_unused_filename($contextid, $component, $filearea, $itemid, $filepath, ''); } + + /** + * Test that mimetype_from_file returns appropriate output when the + * file could not be found. + */ + public function test_mimetype_not_found() { + $mimetype = file_storage::mimetype('/path/to/nonexistent/file'); + $this->assertEquals('document/unknown', $mimetype); + } + + /** + * Test that mimetype_from_file returns appropriate output for a known + * file. + * + * Note: this is not intended to check that functions outside of this + * file works. It is intended to validate the codepath contains no + * errors and behaves as expected. + */ + public function test_mimetype_known() { + $filepath = __DIR__ . DIRECTORY_SEPARATOR . 'fixtures' . DIRECTORY_SEPARATOR . 'testimage.jpg'; + $mimetype = file_storage::mimetype_from_file($filepath); + $this->assertEquals('image/jpeg', $mimetype); + } + + /** + * Test that mimetype_from_file returns appropriate output when the + * file could not be found. + */ + public function test_mimetype_from_file_not_found() { + $mimetype = file_storage::mimetype_from_file('/path/to/nonexistent/file'); + $this->assertEquals('document/unknown', $mimetype); + } + + /** + * Test that mimetype_from_file returns appropriate output for a known + * file. + * + * Note: this is not intended to check that functions outside of this + * file works. It is intended to validate the codepath contains no + * errors and behaves as expected. + */ + public function test_mimetype_from_file_known() { + $filepath = __DIR__ . DIRECTORY_SEPARATOR . 'fixtures' . DIRECTORY_SEPARATOR . 'testimage.jpg'; + $mimetype = file_storage::mimetype_from_file($filepath); + $this->assertEquals('image/jpeg', $mimetype); + } + } class test_stored_file_inspection extends stored_file { diff --git a/lib/filestorage/tests/file_system_filedir_test.php b/lib/filestorage/tests/file_system_filedir_test.php new file mode 100644 index 0000000000000..1f65099cb9d98 --- /dev/null +++ b/lib/filestorage/tests/file_system_filedir_test.php @@ -0,0 +1,1063 @@ +. + +/** + * Unit tests for file_system_filedir. + * + * @package core_files + * @category phpunit + * @copyright 2017 Andrew Nicols + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +defined('MOODLE_INTERNAL') || die(); + +global $CFG; +require_once($CFG->libdir . '/filestorage/file_system.php'); +require_once($CFG->libdir . '/filestorage/file_system_filedir.php'); + +/** + * Unit tests for file_system_filedir. + * + * @package core_files + * @category files + * @copyright 2017 Andrew Nicols + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class core_files_file_system_filedir_testcase extends advanced_testcase { + + /** + * Shared test setUp. + */ + public function setUp() { + // Reset the file storage so that subsequent fetches to get_file_storage are called after + // configuration is prepared. + get_file_storage(true); + } + + /** + * Shared teset tearDown. + */ + public function tearDown() { + // Reset the file storage so that subsequent tests will use the standard file storage. + get_file_storage(true); + } + + /** + * Helper function to help setup and configure the virtual file system stream. + * + * @param array $filedir Directory structure and content of the filedir + * @param array $trashdir Directory structure and content of the sourcedir + * @param array $sourcedir Directory structure and content of a directory used for source files for tests + * @return \org\bovigo\vfs\vfsStream + */ + protected function setup_vfile_root($filedir = [], $trashdir = [], $sourcedir = null) { + global $CFG; + $this->resetAfterTest(); + + $content = []; + if ($filedir !== null) { + $content['filedir'] = $filedir; + } + + if ($trashdir !== null) { + $content['trashdir'] = $trashdir; + } + + if ($sourcedir !== null) { + $content['sourcedir'] = $sourcedir; + } + + $vfileroot = \org\bovigo\vfs\vfsStream::setup('root', null, $content); + + $CFG->filedir = \org\bovigo\vfs\vfsStream::url('root/filedir'); + $CFG->trashdir = \org\bovigo\vfs\vfsStream::url('root/trashdir'); + + return $vfileroot; + } + + /** + * Helper to create a stored file objectw with the given supplied content. + * + * @param string $filecontent The content of the mocked file + * @param string $filename The file name to use in the stored_file + * @param array $mockedmethods A list of methods you intend to override + * If no methods are specified, only abstract functions are mocked. + * @return stored_file + */ + protected function get_stored_file($filecontent, $filename = null, $mockedmethods = null) { + $contenthash = sha1($filecontent); + if (empty($filename)) { + $filename = $contenthash; + } + + $file = $this->getMockBuilder(stored_file::class) + ->setMethods($mockedmethods) + ->setConstructorArgs([ + get_file_storage(), + (object) [ + 'contenthash' => $contenthash, + 'filesize' => strlen($filecontent), + 'filename' => $filename, + ] + ]) + ->getMock(); + + return $file; + } + + /** + * Get a testable mock of the file_system_filedir class. + * + * @param array $mockedmethods A list of methods you intend to override + * If no methods are specified, only abstract functions are mocked. + * @return file_system + */ + protected function get_testable_mock($mockedmethods = []) { + $fs = $this->getMockBuilder(file_system_filedir::class) + ->setMethods($mockedmethods) + ->getMock(); + + return $fs; + } + + /** + * Ensure that an appropriate error is shown when the filedir directory + * is not writable. + */ + public function test_readonly_filesystem_filedir() { + $this->resetAfterTest(); + + // Setup the filedir but remove permissions. + $vfileroot = $this->setup_vfile_root(null); + + // Make the target path readonly. + $vfileroot->chmod(0444) + ->chown(\org\bovigo\vfs\vfsStream::OWNER_USER_2); + + // This should generate an exception. + $this->setExpectedExceptionRegexp('file_exception', + '/Can not create local file pool directories, please verify permissions in dataroot./'); + + new file_system_filedir(); + } + + /** + * Ensure that an appropriate error is shown when the trash directory + * is not writable. + */ + public function test_readonly_filesystem_trashdir() { + $this->resetAfterTest(); + + // Setup the trashdir but remove permissions. + $vfileroot = $this->setup_vfile_root([], null); + + // Make the target path readonly. + $vfileroot->chmod(0444) + ->chown(\org\bovigo\vfs\vfsStream::OWNER_USER_2); + + // This should generate an exception. + $this->setExpectedExceptionRegexp('file_exception', + '/Can not create local file pool directories, please verify permissions in dataroot./'); + + new file_system_filedir(); + } + + /** + * Test that the standard Moodle warning message is put into the filedir. + */ + public function test_warnings_put_in_place() { + $this->resetAfterTest(); + + $vfileroot = $this->setup_vfile_root(null); + + new file_system_filedir(); + $this->assertTrue($vfileroot->hasChild('filedir/warning.txt')); + $this->assertEquals( + 'This directory contains the content of uploaded files and is controlled by Moodle code. ' . + 'Do not manually move, change or rename any of the files and subdirectories here.', + $vfileroot->getChild('filedir/warning.txt')->getContent() + ); + } + + /** + * Ensure that the default implementation of get_remote_path_from_hash + * simply calls get_local_path_from_hash. + */ + public function test_get_remote_path_from_hash() { + $filecontent = 'example content'; + $contenthash = sha1($filecontent); + $expectedresult = (object) []; + + $fs = $this->get_testable_mock([ + 'get_local_path_from_hash', + ]); + + $fs->expects($this->once()) + ->method('get_local_path_from_hash') + ->with($this->equalTo($contenthash), $this->equalTo(false)) + ->willReturn($expectedresult); + + $method = new ReflectionMethod(file_system_filedir::class, 'get_remote_path_from_hash'); + $method->setAccessible(true); + $result = $method->invokeArgs($fs, [$contenthash]); + + $this->assertEquals($expectedresult, $result); + } + + /** + * Test the stock implementation of get_local_path_from_storedfile_with_recovery with no file found and + * a failed recovery. + */ + public function test_get_local_path_from_storedfile_with_recovery() { + $filecontent = 'example content'; + $file = $this->get_stored_file($filecontent); + $fs = $this->get_testable_mock([ + 'get_local_path_from_hash', + 'recover_file', + ]); + $filepath = '/path/to/nonexistent/file'; + + $fs->method('get_local_path_from_hash') + ->willReturn($filepath); + + $fs->expects($this->once()) + ->method('recover_file') + ->with($this->equalTo($file)); + + $file = $this->get_stored_file('example content'); + $method = new ReflectionMethod(file_system_filedir::class, 'get_local_path_from_storedfile'); + $method->setAccessible(true); + $result = $method->invokeArgs($fs, array($file, true)); + + $this->assertEquals($filepath, $result); + } + + /** + * Test the stock implementation of get_local_path_from_storedfile_with_recovery with no file found and + * a failed recovery. + */ + public function test_get_local_path_from_storedfile_without_recovery() { + $filecontent = 'example content'; + $file = $this->get_stored_file($filecontent); + $fs = $this->get_testable_mock([ + 'get_local_path_from_hash', + 'recover_file', + ]); + $filepath = '/path/to/nonexistent/file'; + + $fs->method('get_local_path_from_hash') + ->willReturn($filepath); + + $fs->expects($this->never()) + ->method('recover_file'); + + $file = $this->get_stored_file('example content'); + $method = new ReflectionMethod(file_system_filedir::class, 'get_local_path_from_storedfile'); + $method->setAccessible(true); + $result = $method->invokeArgs($fs, array($file, false)); + + $this->assertEquals($filepath, $result); + } + + /** + * Test that the correct path is generated for the supplied content + * hashes. + * + * @dataProvider contenthash_dataprovider + * @param string $hash contenthash to test + * @param string $hashdir Expected format of content directory + */ + public function test_get_fulldir_from_hash($hash, $hashdir) { + global $CFG; + + $fs = new file_system_filedir(); + $method = new ReflectionMethod(file_system_filedir::class, 'get_fulldir_from_hash'); + $method->setAccessible(true); + $result = $method->invokeArgs($fs, array($hash)); + + $expectedpath = sprintf('%s/filedir/%s', $CFG->dataroot, $hashdir); + $this->assertEquals($expectedpath, $result); + } + + /** + * Test that the correct path is generated for the supplied content + * hashes when used with a stored_file. + * + * @dataProvider contenthash_dataprovider + * @param string $hash contenthash to test + * @param string $hashdir Expected format of content directory + */ + public function test_get_fulldir_from_storedfile($hash, $hashdir) { + global $CFG; + + $file = $this->getMockBuilder('stored_file') + ->disableOriginalConstructor() + ->setMethods([ + 'sync_external_file', + 'get_contenthash', + ]) + ->getMock(); + + $file->method('get_contenthash')->willReturn($hash); + + $fs = new file_system_filedir(); + $method = new ReflectionMethod('file_system_filedir', 'get_fulldir_from_storedfile'); + $method->setAccessible(true); + $result = $method->invokeArgs($fs, array($file)); + + $expectedpath = sprintf('%s/filedir/%s', $CFG->dataroot, $hashdir); + $this->assertEquals($expectedpath, $result); + } + + /** + * Test that the correct content directory is generated for the supplied + * content hashes. + * + * @dataProvider contenthash_dataprovider + * @param string $hash contenthash to test + * @param string $hashdir Expected format of content directory + */ + public function test_get_contentdir_from_hash($hash, $hashdir) { + $method = new ReflectionMethod(file_system_filedir::class, 'get_contentdir_from_hash'); + $method->setAccessible(true); + + $fs = new file_system_filedir(); + $result = $method->invokeArgs($fs, array($hash)); + + $this->assertEquals($hashdir, $result); + } + + /** + * Test that the correct content path is generated for the supplied + * content hashes. + * + * @dataProvider contenthash_dataprovider + * @param string $hash contenthash to test + * @param string $hashdir Expected format of content directory + */ + public function test_get_contentpath_from_hash($hash, $hashdir) { + $method = new ReflectionMethod(file_system_filedir::class, 'get_contentpath_from_hash'); + $method->setAccessible(true); + + $fs = new file_system_filedir(); + $result = $method->invokeArgs($fs, array($hash)); + + $expectedpath = sprintf('%s/%s', $hashdir, $hash); + $this->assertEquals($expectedpath, $result); + } + + /** + * Test that the correct trash path is generated for the supplied + * content hashes. + * + * @dataProvider contenthash_dataprovider + * @param string $hash contenthash to test + * @param string $hashdir Expected format of content directory + */ + public function test_get_trash_fullpath_from_hash($hash, $hashdir) { + global $CFG; + + $fs = new file_system_filedir(); + $method = new ReflectionMethod(file_system_filedir::class, 'get_trash_fullpath_from_hash'); + $method->setAccessible(true); + $result = $method->invokeArgs($fs, array($hash)); + + $expectedpath = sprintf('%s/trashdir/%s/%s', $CFG->dataroot, $hashdir, $hash); + $this->assertEquals($expectedpath, $result); + } + + /** + * Test that the correct trash directory is generated for the supplied + * content hashes. + * + * @dataProvider contenthash_dataprovider + * @param string $hash contenthash to test + * @param string $hashdir Expected format of content directory + */ + public function test_get_trash_fulldir_from_hash($hash, $hashdir) { + global $CFG; + + $fs = new file_system_filedir(); + $method = new ReflectionMethod(file_system_filedir::class, 'get_trash_fulldir_from_hash'); + $method->setAccessible(true); + $result = $method->invokeArgs($fs, array($hash)); + + $expectedpath = sprintf('%s/trashdir/%s', $CFG->dataroot, $hashdir); + $this->assertEquals($expectedpath, $result); + } + + /** + * Ensure that copying a file to a target from a stored_file works as anticipated. + */ + public function test_copy_content_from_storedfile() { + $this->resetAfterTest(); + global $CFG; + + $filecontent = 'example content'; + $contenthash = sha1($filecontent); + $filedircontent = [ + $contenthash => $filecontent, + ]; + $vfileroot = $this->setup_vfile_root($filedircontent, [], []); + + $fs = $this->getMockBuilder(file_system_filedir::class) + ->disableOriginalConstructor() + ->setMethods([ + 'get_local_path_from_storedfile', + ]) + ->getMock(); + + $file = $this->getMockBuilder(stored_file::class) + ->disableOriginalConstructor() + ->getMock(); + + $sourcefile = \org\bovigo\vfs\vfsStream::url('root/filedir/' . $contenthash); + $fs->method('get_local_path_from_storedfile')->willReturn($sourcefile); + + $targetfile = \org\bovigo\vfs\vfsStream::url('root/targetfile'); + $CFG->preventfilelocking = true; + $result = $fs->copy_content_from_storedfile($file, $targetfile); + + $this->assertTrue($result); + $this->assertEquals($filecontent, $vfileroot->getChild('targetfile')->getContent()); + } + + /** + * Ensure that content recovery works. + */ + public function test_recover_file() { + $this->resetAfterTest(); + + // Setup the filedir. + // This contains a virtual file which has a cache mismatch. + $filecontent = 'example content'; + $contenthash = sha1($filecontent); + + $trashdircontent = [ + '0f' => [ + 'f3' => [ + $contenthash => $filecontent, + ], + ], + ]; + + $vfileroot = $this->setup_vfile_root([], $trashdircontent); + + $file = new stored_file(get_file_storage(), (object) [ + 'contenthash' => $contenthash, + 'filesize' => strlen($filecontent), + ]); + + $fs = new file_system_filedir(); + $method = new ReflectionMethod(file_system_filedir::class, 'recover_file'); + $method->setAccessible(true); + $result = $method->invokeArgs($fs, array($file)); + + // Test the output. + $this->assertTrue($result); + + $this->assertEquals($filecontent, $vfileroot->getChild('filedir/0f/f3/' . $contenthash)->getContent()); + + } + + /** + * Ensure that content recovery works. + */ + public function test_recover_file_already_present() { + $this->resetAfterTest(); + + // Setup the filedir. + // This contains a virtual file which has a cache mismatch. + $filecontent = 'example content'; + $contenthash = sha1($filecontent); + + $filedircontent = $trashdircontent = [ + '0f' => [ + 'f3' => [ + $contenthash => $filecontent, + ], + ], + ]; + + $vfileroot = $this->setup_vfile_root($filedircontent, $trashdircontent); + + $file = new stored_file(get_file_storage(), (object) [ + 'contenthash' => $contenthash, + 'filesize' => strlen($filecontent), + ]); + + $fs = new file_system_filedir(); + $method = new ReflectionMethod(file_system_filedir::class, 'recover_file'); + $method->setAccessible(true); + $result = $method->invokeArgs($fs, array($file)); + + // Test the output. + $this->assertTrue($result); + + $this->assertEquals($filecontent, $vfileroot->getChild('filedir/0f/f3/' . $contenthash)->getContent()); + } + + /** + * Ensure that content recovery works. + */ + public function test_recover_file_size_mismatch() { + $this->resetAfterTest(); + + // Setup the filedir. + // This contains a virtual file which has a cache mismatch. + $filecontent = 'example content'; + $contenthash = sha1($filecontent); + + $trashdircontent = [ + '0f' => [ + 'f3' => [ + $contenthash => $filecontent, + ], + ], + ]; + $vfileroot = $this->setup_vfile_root([], $trashdircontent); + + $file = new stored_file(get_file_storage(), (object) [ + 'contenthash' => $contenthash, + 'filesize' => strlen($filecontent) + 1, + ]); + + $fs = new file_system_filedir(); + $method = new ReflectionMethod(file_system_filedir::class, 'recover_file'); + $method->setAccessible(true); + $result = $method->invokeArgs($fs, array($file)); + + // Test the output. + $this->assertFalse($result); + $this->assertFalse($vfileroot->hasChild('filedir/0f/f3/' . $contenthash)); + } + + /** + * Ensure that content recovery works. + */ + public function test_recover_file_has_mismatch() { + $this->resetAfterTest(); + + // Setup the filedir. + // This contains a virtual file which has a cache mismatch. + $filecontent = 'example content'; + $contenthash = sha1($filecontent); + + $trashdircontent = [ + '0f' => [ + 'f3' => [ + $contenthash => $filecontent, + ], + ], + ]; + $vfileroot = $this->setup_vfile_root([], $trashdircontent); + + $file = new stored_file(get_file_storage(), (object) [ + 'contenthash' => $contenthash . " different", + 'filesize' => strlen($filecontent), + ]); + + $fs = new file_system_filedir(); + $method = new ReflectionMethod(file_system_filedir::class, 'recover_file'); + $method->setAccessible(true); + $result = $method->invokeArgs($fs, array($file)); + + // Test the output. + $this->assertFalse($result); + $this->assertFalse($vfileroot->hasChild('filedir/0f/f3/' . $contenthash)); + } + + /** + * Ensure that content recovery works when the content file is in the + * alt trash directory. + */ + public function test_recover_file_alttrash() { + $this->resetAfterTest(); + + // Setup the filedir. + // This contains a virtual file which has a cache mismatch. + $filecontent = 'example content'; + $contenthash = sha1($filecontent); + + $trashdircontent = [ + $contenthash => $filecontent, + ]; + $vfileroot = $this->setup_vfile_root([], $trashdircontent); + + $file = new stored_file(get_file_storage(), (object) [ + 'contenthash' => $contenthash, + 'filesize' => strlen($filecontent), + ]); + + $fs = new file_system_filedir(); + $method = new ReflectionMethod(file_system_filedir::class, 'recover_file'); + $method->setAccessible(true); + $result = $method->invokeArgs($fs, array($file)); + + // Test the output. + $this->assertTrue($result); + + $this->assertEquals($filecontent, $vfileroot->getChild('filedir/0f/f3/' . $contenthash)->getContent()); + } + + /** + * Test that an appropriate error message is generated when adding a + * file to the pool when the pool directory structure is not writable. + */ + public function test_recover_file_contentdir_readonly() { + $this->resetAfterTest(); + + $filecontent = 'example content'; + $contenthash = sha1($filecontent); + $filedircontent = [ + '0f' => [], + ]; + $trashdircontent = [ + $contenthash => $filecontent, + ]; + $vfileroot = $this->setup_vfile_root($filedircontent, $trashdircontent); + + // Make the target path readonly. + $vfileroot->getChild('filedir/0f') + ->chmod(0444) + ->chown(\org\bovigo\vfs\vfsStream::OWNER_USER_2); + + $file = new stored_file(get_file_storage(), (object) [ + 'contenthash' => $contenthash, + 'filesize' => strlen($filecontent), + ]); + + $fs = new file_system_filedir(); + $method = new ReflectionMethod(file_system_filedir::class, 'recover_file'); + $method->setAccessible(true); + $result = $method->invokeArgs($fs, array($file)); + + // Test the output. + $this->assertFalse($result); + } + + /** + * Test adding a file to the pool. + */ + public function test_add_file_from_path() { + $this->resetAfterTest(); + global $CFG; + + // Setup the filedir. + // This contains a virtual file which has a cache mismatch. + $filecontent = 'example content'; + $contenthash = sha1($filecontent); + $sourcedircontent = [ + 'file' => $filecontent, + ]; + + $vfileroot = $this->setup_vfile_root([], [], $sourcedircontent); + + // Note, the vfs file system does not support locks - prevent file locking here. + $CFG->preventfilelocking = true; + + // Attempt to add the file to the file pool. + $fs = new file_system_filedir(); + $sourcefile = \org\bovigo\vfs\vfsStream::url('root/sourcedir/file'); + $result = $fs->add_file_from_path($sourcefile); + + // Test the output. + $this->assertEquals($contenthash, $result[0]); + $this->assertEquals(core_text::strlen($filecontent), $result[1]); + $this->assertTrue($result[2]); + + $this->assertEquals($filecontent, $vfileroot->getChild('filedir/0f/f3/' . $contenthash)->getContent()); + } + + /** + * Test that an appropriate error message is generated when adding an + * unavailable file to the pool is attempted. + */ + public function test_add_file_from_path_file_unavailable() { + $this->resetAfterTest(); + + // Setup the filedir. + $vfileroot = $this->setup_vfile_root(); + + $this->setExpectedExceptionRegexp('file_exception', + '/Cannot read file\. Either the file does not exist or there is a permission problem\./'); + + $fs = new file_system_filedir(); + $fs->add_file_from_path(\org\bovigo\vfs\vfsStream::url('filedir/file')); + } + + /** + * Test that an appropriate error message is generated when specifying + * the wrong contenthash when adding a file to the pool. + */ + public function test_add_file_from_path_mismatched_hash() { + $this->resetAfterTest(); + + $filecontent = 'example content'; + $contenthash = sha1($filecontent); + $sourcedir = [ + 'file' => $filecontent, + ]; + $vfileroot = $this->setup_vfile_root([], [], $sourcedir); + + $fs = new file_system_filedir(); + $filepath = \org\bovigo\vfs\vfsStream::url('root/sourcedir/file'); + $fs->add_file_from_path($filepath, 'eee4943847a35a4b6942c6f96daafde06bcfdfab'); + $this->assertDebuggingCalled("Invalid contenthash submitted for file $filepath"); + } + + /** + * Test that an appropriate error message is generated when an existing + * file in the pool has the wrong contenthash + */ + public function test_add_file_from_path_existing_content_invalid() { + $this->resetAfterTest(); + + $filecontent = 'example content'; + $contenthash = sha1($filecontent); + $filedircontent = [ + '0f' => [ + 'f3' => [ + // This contains a virtual file which has a cache mismatch. + '0ff30941ca5acd879fd809e8c937d9f9e6dd1615' => 'different example content', + ], + ], + ]; + $sourcedir = [ + 'file' => $filecontent, + ]; + $vfileroot = $this->setup_vfile_root($filedircontent, [], $sourcedir); + + // Check that we hit the jackpot. + $fs = new file_system_filedir(); + $filepath = \org\bovigo\vfs\vfsStream::url('root/sourcedir/file'); + $result = $fs->add_file_from_path($filepath); + + // We provided a bad hash. Check that the file was replaced. + $this->assertDebuggingCalled("Replacing invalid content file $contenthash"); + + // Test the output. + $this->assertEquals($contenthash, $result[0]); + $this->assertEquals(core_text::strlen($filecontent), $result[1]); + $this->assertFalse($result[2]); + + // Fetch the new file structure. + $structure = \org\bovigo\vfs\vfsStream::inspect( + new \org\bovigo\vfs\visitor\vfsStreamStructureVisitor() + )->getStructure(); + + $this->assertEquals($filecontent, $structure['root']['filedir']['0f']['f3'][$contenthash]); + } + + /** + * Test that an appropriate error message is generated when adding a + * file to the pool when the pool directory structure is not writable. + */ + public function test_add_file_from_path_existing_cannot_write_hashpath() { + $this->resetAfterTest(); + + $filecontent = 'example content'; + $contenthash = sha1($filecontent); + $filedircontent = [ + '0f' => [], + ]; + $sourcedir = [ + 'file' => $filecontent, + ]; + $vfileroot = $this->setup_vfile_root($filedircontent, [], $sourcedir); + + // Make the target path readonly. + $vfileroot->getChild('filedir/0f') + ->chmod(0444) + ->chown(\org\bovigo\vfs\vfsStream::OWNER_USER_2); + + $this->setExpectedException( + 'file_exception', + "Can not create local file pool directories, please verify permissions in dataroot." + ); + + // Attempt to add the file to the file pool. + $fs = new file_system_filedir(); + $sourcefile = \org\bovigo\vfs\vfsStream::url('root/sourcedir/file'); + $fs->add_file_from_path($sourcefile); + } + + /** + * Test adding a string to the pool. + */ + public function test_add_file_from_string() { + $this->resetAfterTest(); + global $CFG; + + $filecontent = 'example content'; + $contenthash = sha1($filecontent); + $vfileroot = $this->setup_vfile_root(); + + // Note, the vfs file system does not support locks - prevent file locking here. + $CFG->preventfilelocking = true; + + // Attempt to add the file to the file pool. + $fs = new file_system_filedir(); + $result = $fs->add_file_from_string($filecontent); + + // Test the output. + $this->assertEquals($contenthash, $result[0]); + $this->assertEquals(core_text::strlen($filecontent), $result[1]); + $this->assertTrue($result[2]); + } + + /** + * Test that an appropriate error message is generated when adding a + * string to the pool when the pool directory structure is not writable. + */ + public function test_add_file_from_string_existing_cannot_write_hashpath() { + $this->resetAfterTest(); + + $filecontent = 'example content'; + $contenthash = sha1($filecontent); + + $filedircontent = [ + '0f' => [], + ]; + $vfileroot = $this->setup_vfile_root($filedircontent); + + // Make the target path readonly. + $vfileroot->getChild('filedir/0f') + ->chmod(0444) + ->chown(\org\bovigo\vfs\vfsStream::OWNER_USER_2); + + $this->setExpectedException( + 'file_exception', + "Can not create local file pool directories, please verify permissions in dataroot." + ); + + // Attempt to add the file to the file pool. + $fs = new file_system_filedir(); + $fs->add_file_from_string($filecontent); + } + + /** + * Test adding a string to the pool when an item with the same + * contenthash is already present. + */ + public function test_add_file_from_string_existing_matches() { + $this->resetAfterTest(); + global $CFG; + + $filecontent = 'example content'; + $contenthash = sha1($filecontent); + $filedircontent = [ + '0f' => [ + 'f3' => [ + $contenthash => $filecontent, + ], + ], + ]; + + $vfileroot = $this->setup_vfile_root($filedircontent); + + // Note, the vfs file system does not support locks - prevent file locking here. + $CFG->preventfilelocking = true; + + // Attempt to add the file to the file pool. + $fs = new file_system_filedir(); + $result = $fs->add_file_from_string($filecontent); + + // Test the output. + $this->assertEquals($contenthash, $result[0]); + $this->assertEquals(core_text::strlen($filecontent), $result[1]); + $this->assertFalse($result[2]); + } + + /** + * Test the cleanup of deleted files when there are no files to delete. + */ + public function test_remove_file_missing() { + $this->resetAfterTest(); + + $filecontent = 'example content'; + $contenthash = sha1($filecontent); + $vfileroot = $this->setup_vfile_root(); + + $fs = new file_system_filedir(); + $fs->remove_file($contenthash); + + $this->assertFalse($vfileroot->hasChild('filedir/0f/f3/' . $contenthash)); + // No file to move to trash, so the trash path will also be empty. + $this->assertFalse($vfileroot->hasChild('trashdir/0f')); + $this->assertFalse($vfileroot->hasChild('trashdir/0f/f3')); + $this->assertFalse($vfileroot->hasChild('trashdir/0f/f3/' . $contenthash)); + } + + /** + * Test the cleanup of deleted files when a file already exists in the + * trash for that path. + */ + public function test_remove_file_existing_trash() { + $this->resetAfterTest(); + + $filecontent = 'example content'; + $contenthash = sha1($filecontent); + + $filedircontent = $trashdircontent = [ + '0f' => [ + 'f3' => [ + $contenthash => $filecontent, + ], + ], + ]; + $trashdircontent['0f']['f3'][$contenthash] .= 'different'; + $vfileroot = $this->setup_vfile_root($filedircontent, $trashdircontent); + + $fs = new file_system_filedir(); + $fs->remove_file($contenthash); + + $this->assertFalse($vfileroot->hasChild('filedir/0f/f3/' . $contenthash)); + $this->assertTrue($vfileroot->hasChild('trashdir/0f/f3/' . $contenthash)); + $this->assertNotEquals($filecontent, $vfileroot->getChild('trashdir/0f/f3/' . $contenthash)->getContent()); + } + + /** + * Ensure that remove_file does nothing with an empty file. + */ + public function test_remove_file_empty() { + $this->resetAfterTest(); + global $DB; + + $DB = $this->getMockBuilder(\moodle_database::class) + ->setMethods(['record_exists']) + ->getMockForAbstractClass(); + + $DB->expects($this->never()) + ->method('record_exists'); + + $fs = new file_system_filedir(); + + $result = $fs->remove_file(sha1('')); + $this->assertNull($result); + } + + /** + * Ensure that remove_file does nothing when a file is still + * in use. + */ + public function test_remove_file_in_use() { + $this->resetAfterTest(); + global $DB; + + $filecontent = 'example content'; + $contenthash = sha1($filecontent); + $filedircontent = [ + '0f' => [ + 'f3' => [ + $contenthash => $filecontent, + ], + ], + ]; + $vfileroot = $this->setup_vfile_root($filedircontent); + + $DB = $this->getMockBuilder(\moodle_database::class) + ->setMethods(['record_exists']) + ->getMockForAbstractClass(); + + $DB->method('record_exists')->willReturn(true); + + $fs = new file_system_filedir(); + $result = $fs->remove_file($contenthash); + $this->assertTrue($vfileroot->hasChild('filedir/0f/f3/' . $contenthash)); + $this->assertFalse($vfileroot->hasChild('trashdir/0f/f3/' . $contenthash)); + } + + /** + * Ensure that remove_file removes the file when it is no + * longer in use. + */ + public function test_remove_file_expired() { + $this->resetAfterTest(); + global $DB; + + $filecontent = 'example content'; + $contenthash = sha1($filecontent); + $filedircontent = [ + '0f' => [ + 'f3' => [ + $contenthash => $filecontent, + ], + ], + ]; + $vfileroot = $this->setup_vfile_root($filedircontent); + + $DB = $this->getMockBuilder(\moodle_database::class) + ->setMethods(['record_exists']) + ->getMockForAbstractClass(); + + $DB->method('record_exists')->willReturn(false); + + $fs = new file_system_filedir(); + $result = $fs->remove_file($contenthash); + $this->assertFalse($vfileroot->hasChild('filedir/0f/f3/' . $contenthash)); + $this->assertTrue($vfileroot->hasChild('trashdir/0f/f3/' . $contenthash)); + } + + /** + * Test purging the cache. + */ + public function test_empty_trash() { + $this->resetAfterTest(); + + $filecontent = 'example content'; + $contenthash = sha1($filecontent); + + $filedircontent = $trashdircontent = [ + '0f' => [ + 'f3' => [ + $contenthash => $filecontent, + ], + ], + ]; + $vfileroot = $this->setup_vfile_root($filedircontent, $trashdircontent); + + $fs = new file_system_filedir(); + $method = new ReflectionMethod(file_system_filedir::class, 'empty_trash'); + $method->setAccessible(true); + $result = $method->invoke($fs); + + $this->assertTrue($vfileroot->hasChild('filedir/0f/f3/' . $contenthash)); + $this->assertFalse($vfileroot->hasChild('trashdir')); + $this->assertFalse($vfileroot->hasChild('trashdir/0f')); + $this->assertFalse($vfileroot->hasChild('trashdir/0f/f3')); + $this->assertFalse($vfileroot->hasChild('trashdir/0f/f3/' . $contenthash)); + } + + /** + * Data Provider for contenthash to contendir conversion. + * + * @return array + */ + public function contenthash_dataprovider() { + return array( + array( + 'contenthash' => 'eee4943847a35a4b6942c6f96daafde06bcfdfab', + 'contentdir' => 'ee/e4', + ), + array( + 'contenthash' => 'aef05a62ae81ca0005d2569447779af062b7cda0', + 'contentdir' => 'ae/f0', + ), + ); + } +} diff --git a/lib/filestorage/tests/file_system_test.php b/lib/filestorage/tests/file_system_test.php new file mode 100644 index 0000000000000..2aeff76edab3f --- /dev/null +++ b/lib/filestorage/tests/file_system_test.php @@ -0,0 +1,1091 @@ +. + +/** + * Unit tests for file_system. + * + * @package core_files + * @category phpunit + * @copyright 2017 Andrew Nicols + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +defined('MOODLE_INTERNAL') || die(); + +global $CFG; +require_once($CFG->libdir . '/filestorage/file_system.php'); + +/** + * Unit tests for file_system. + * + * @package core_files + * @category phpunit + * @copyright 2017 Andrew Nicols + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class core_files_file_system_testcase extends advanced_testcase { + + public function setUp() { + get_file_storage(true); + } + + public function tearDown() { + get_file_storage(true); + } + + /** + * Helper function to help setup and configure the virtual file system stream. + * + * @param array $filedir Directory structure and content of the filedir + * @param array $trashdir Directory structure and content of the sourcedir + * @param array $sourcedir Directory structure and content of a directory used for source files for tests + * @return \org\bovigo\vfs\vfsStream + */ + protected function setup_vfile_root($content = []) { + $vfileroot = \org\bovigo\vfs\vfsStream::setup('root', null, $content); + + return $vfileroot; + } + + /** + * Helper to create a stored file objectw with the given supplied content. + * + * @param string $filecontent The content of the mocked file + * @param string $filename The file name to use in the stored_file + * @param array $mockedmethods A list of methods you intend to override + * If no methods are specified, only abstract functions are mocked. + * @return stored_file + */ + protected function get_stored_file($filecontent, $filename = null, $mockedmethods = null) { + $contenthash = sha1($filecontent); + if (empty($filename)) { + $filename = $contenthash; + } + + $file = $this->getMockBuilder(stored_file::class) + ->setMethods($mockedmethods) + ->setConstructorArgs([ + get_file_storage(), + (object) [ + 'contenthash' => $contenthash, + 'filesize' => strlen($filecontent), + 'filename' => $filename, + ] + ]) + ->getMock(); + + return $file; + } + + /** + * Get a testable mock of the abstract file_system class. + * + * @param array $mockedmethods A list of methods you intend to override + * If no methods are specified, only abstract functions are mocked. + * @return file_system + */ + protected function get_testable_mock($mockedmethods = []) { + $fs = $this->getMockBuilder(file_system::class) + ->setMethods($mockedmethods) + ->getMockForAbstractClass(); + + return $fs; + } + + /** + * Ensure that the file system is not clonable. + */ + public function test_not_cloneable() { + $reflection = new ReflectionClass('file_system'); + $this->assertFalse($reflection->isCloneable()); + } + + /** + * Ensure that the filedir file_system extension is used by default. + */ + public function test_default_class() { + $this->resetAfterTest(); + + // Ensure that the alternative_file_system_class is null. + global $CFG; + $CFG->alternative_file_system_class = null; + + $storage = get_file_storage(); + $fs = $storage->get_file_system(); + $this->assertInstanceOf(file_system::class, $fs); + $this->assertEquals(file_system_filedir::class, get_class($fs)); + } + + /** + * Ensure that the specified file_system extension class is used. + */ + public function test_supplied_class() { + global $CFG; + $this->resetAfterTest(); + + // Mock the file_system. + // Mocks create a new child of the mocked class which is perfect for this test. + $filesystem = $this->getMockBuilder('file_system') + ->disableOriginalConstructor() + ->getMock(); + $CFG->alternative_file_system_class = get_class($filesystem); + + $storage = get_file_storage(); + $fs = $storage->get_file_system(); + $this->assertInstanceOf(file_system::class, $fs); + $this->assertEquals(get_class($filesystem), get_class($fs)); + } + + /** + * Test that the readfile function outputs content to disk. + */ + public function test_readfile_remote() { + global $CFG; + + // Mock the filesystem. + $filecontent = 'example content'; + $vfileroot = $this->setup_vfile_root(['sourcefile' => $filecontent]); + $filepath = \org\bovigo\vfs\vfsStream::url('root/sourcefile'); + + $file = $this->get_stored_file($filecontent); + + // Mock the file_system class. + // We need to override the get_remote_path_from_storedfile function. + $fs = $this->get_testable_mock([ + 'get_remote_path_from_storedfile', + 'is_file_readable_locally_by_storedfile', + 'get_local_path_from_storedfile', + ]); + $fs->method('get_remote_path_from_storedfile')->willReturn($filepath); + $fs->method('is_file_readable_locally_by_storedfile')->willReturn(false); + $fs->expects($this->never())->method('get_local_path_from_storedfile'); + + // Note: It is currently not possible to mock readfile_allow_large + // because file_system is in the global namespace. + // We must therefore check for expected output. This is not ideal. + $this->expectOutputString($filecontent); + $fs->readfile($file); + } + + /** + * Test that the readfile function outputs content to disk. + */ + public function test_readfile_local() { + global $CFG; + + // Mock the filesystem. + $filecontent = 'example content'; + $vfileroot = $this->setup_vfile_root(['sourcefile' => $filecontent]); + $filepath = \org\bovigo\vfs\vfsStream::url('root/sourcefile'); + + $file = $this->get_stored_file($filecontent); + + // Mock the file_system class. + // We need to override the get_remote_path_from_storedfile function. + $fs = $this->get_testable_mock([ + 'get_remote_path_from_storedfile', + 'is_file_readable_locally_by_storedfile', + 'get_local_path_from_storedfile', + ]); + $fs->method('is_file_readable_locally_by_storedfile')->willReturn(true); + $fs->expects($this->never())->method('get_remote_path_from_storedfile'); + $fs->expects($this->once())->method('get_local_path_from_storedfile')->willReturn($filepath); + + // Note: It is currently not possible to mock readfile_allow_large + // because file_system is in the global namespace. + // We must therefore check for expected output. This is not ideal. + $this->expectOutputString($filecontent); + $fs->readfile($file); + } + + /** + * Test that the get_local_path_from_storedfile function functions + * correctly when called with various args. + * + * @dataProvider get_local_path_from_storedfile_provider + * @param array $args The additional args to pass to get_local_path_from_storedfile + * @param bool $fetch Whether the combination of args should have caused a fetch + */ + public function test_get_local_path_from_storedfile($args, $fetch) { + $filepath = '/path/to/file'; + $filecontent = 'example content'; + + // Get the filesystem mock. + $fs = $this->get_testable_mock([ + 'get_local_path_from_hash', + ]); + $fs->expects($this->once()) + ->method('get_local_path_from_hash') + ->with($this->equalTo(sha1($filecontent)), $this->equalTo($fetch)) + ->willReturn($filepath); + + $file = $this->get_stored_file($filecontent); + + $method = new ReflectionMethod(file_system::class, 'get_local_path_from_storedfile'); + $method->setAccessible(true); + $result = $method->invokeArgs($fs, array_merge([$file], $args)); + + $this->assertEquals($filepath, $result); + } + + /** + * Ensure that the default implementation of get_remote_path_from_storedfile + * simply calls get_local_path_from_storedfile without requiring a + * fetch. + */ + public function test_get_remote_path_from_storedfile() { + $filepath = '/path/to/file'; + $filecontent = 'example content'; + + $fs = $this->get_testable_mock([ + 'get_remote_path_from_hash', + ]); + + $fs->expects($this->once()) + ->method('get_remote_path_from_hash') + ->with($this->equalTo(sha1($filecontent)), $this->equalTo(false)) + ->willReturn($filepath); + + $file = $this->get_stored_file($filecontent); + + $method = new ReflectionMethod(file_system::class, 'get_remote_path_from_storedfile'); + $method->setAccessible(true); + $result = $method->invokeArgs($fs, [$file]); + + $this->assertEquals($filepath, $result); + } + + /** + * Test the stock implementation of is_file_readable_locally_by_hash with a valid file. + * + * This should call get_local_path_from_hash and check the readability + * of the file. + * + * Fetching the file is optional. + */ + public function test_is_file_readable_locally_by_hash() { + $filecontent = 'example content'; + $contenthash = sha1($filecontent); + $filepath = __FILE__; + + $fs = $this->get_testable_mock([ + 'get_local_path_from_hash', + ]); + + $fs->method('get_local_path_from_hash') + ->with($this->equalTo($contenthash), $this->equalTo(false)) + ->willReturn($filepath); + + $this->assertTrue($fs->is_file_readable_locally_by_hash($contenthash)); + } + + /** + * Test the stock implementation of is_file_readable_locally_by_hash with an empty file. + */ + public function test_is_file_readable_locally_by_hash_empty() { + $filecontent = ''; + $contenthash = sha1($filecontent); + + $fs = $this->get_testable_mock([ + 'get_local_path_from_hash', + ]); + + $fs->expects($this->never()) + ->method('get_local_path_from_hash'); + + $this->assertTrue($fs->is_file_readable_locally_by_hash($contenthash)); + } + + /** + * Test the stock implementation of is_file_readable_remotely_by_storedfile with a valid file. + */ + public function test_is_file_readable_remotely_by_hash() { + $filecontent = 'example content'; + $contenthash = sha1($filecontent); + + $fs = $this->get_testable_mock([ + 'get_remote_path_from_hash', + ]); + + $fs->method('get_remote_path_from_hash') + ->with($this->equalTo($contenthash), $this->equalTo(false)) + ->willReturn(__FILE__); + + $this->assertTrue($fs->is_file_readable_remotely_by_hash($contenthash)); + } + + /** + * Test the stock implementation of is_file_readable_remotely_by_storedfile with a valid file. + */ + public function test_is_file_readable_remotely_by_hash_empty() { + $filecontent = ''; + $contenthash = sha1($filecontent); + + $fs = $this->get_testable_mock([ + 'get_remote_path_from_hash', + ]); + + $fs->expects($this->never()) + ->method('get_remote_path_from_hash'); + + $this->assertTrue($fs->is_file_readable_remotely_by_hash($contenthash)); + } + + /** + * Test the stock implementation of is_file_readable_remotely_by_storedfile with a valid file. + */ + public function test_is_file_readable_remotely_by_hash_not_found() { + $filecontent = 'example content'; + $contenthash = sha1($filecontent); + + $fs = $this->get_testable_mock([ + 'get_remote_path_from_hash', + ]); + + $fs->method('get_remote_path_from_hash') + ->with($this->equalTo($contenthash), $this->equalTo(false)) + ->willReturn('/path/to/nonexistent/file'); + + $this->assertFalse($fs->is_file_readable_remotely_by_hash($contenthash)); + } + + /** + * Test the stock implementation of is_file_readable_remotely_by_storedfile with a valid file. + */ + public function test_is_file_readable_remotely_by_storedfile() { + $file = $this->get_stored_file('example content'); + + $fs = $this->get_testable_mock([ + 'get_remote_path_from_storedfile', + ]); + + $fs->method('get_remote_path_from_storedfile') + ->willReturn(__FILE__); + + $this->assertTrue($fs->is_file_readable_remotely_by_storedfile($file)); + } + + /** + * Test the stock implementation of is_file_readable_remotely_by_storedfile with a valid file. + */ + public function test_is_file_readable_remotely_by_storedfile_empty() { + $fs = $this->get_testable_mock([ + 'get_remote_path_from_storedfile', + ]); + + $fs->expects($this->never()) + ->method('get_remote_path_from_storedfile'); + + $file = $this->get_stored_file(''); + $this->assertTrue($fs->is_file_readable_remotely_by_storedfile($file)); + } + + /** + * Test the stock implementation of is_file_readable_locally_by_storedfile with an empty file. + */ + public function test_is_file_readable_locally_by_storedfile_empty() { + $fs = $this->get_testable_mock([ + 'get_local_path_from_storedfile', + ]); + + $fs->expects($this->never()) + ->method('get_local_path_from_storedfile'); + + $file = $this->get_stored_file(''); + $this->assertTrue($fs->is_file_readable_locally_by_storedfile($file)); + } + + /** + * Test the stock implementation of is_file_readable_remotely_by_storedfile with a valid file. + */ + public function test_is_file_readable_remotely_by_storedfile_not_found() { + $file = $this->get_stored_file('example content'); + + $fs = $this->get_testable_mock([ + 'get_remote_path_from_storedfile', + ]); + + $fs->method('get_remote_path_from_storedfile') + ->willReturn(__LINE__); + + $this->assertFalse($fs->is_file_readable_remotely_by_storedfile($file)); + } + + /** + * Test the stock implementation of is_file_readable_locally_by_storedfile with a valid file. + */ + public function test_is_file_readable_locally_by_storedfile_unreadable() { + $fs = $this->get_testable_mock([ + 'get_local_path_from_storedfile', + ]); + $file = $this->get_stored_file('example content'); + + $fs->method('get_local_path_from_storedfile') + ->with($this->equalTo($file), $this->equalTo(false)) + ->willReturn('/path/to/nonexistent/file'); + + $this->assertFalse($fs->is_file_readable_locally_by_storedfile($file)); + } + + /** + * Test the stock implementation of is_file_readable_locally_by_storedfile with a valid file should pass fetch. + */ + public function test_is_file_readable_locally_by_storedfile_passes_fetch() { + $fs = $this->get_testable_mock([ + 'get_local_path_from_storedfile', + ]); + $file = $this->get_stored_file('example content'); + + $fs->method('get_local_path_from_storedfile') + ->with($this->equalTo($file), $this->equalTo(true)) + ->willReturn('/path/to/nonexistent/file'); + + $this->assertFalse($fs->is_file_readable_locally_by_storedfile($file, true)); + } + + /** + * Ensure that is_file_removable returns correctly for an empty file. + */ + public function test_is_file_removable_empty() { + $filecontent = ''; + $contenthash = sha1($filecontent); + + $method = new ReflectionMethod(file_system::class, 'is_file_removable'); + $method->setAccessible(true); + $result = $method->invokeArgs(null, [$contenthash]); + $this->assertFalse($result); + } + + /** + * Ensure that is_file_removable returns false if the file is still in use. + */ + public function test_is_file_removable_in_use() { + $this->resetAfterTest(); + global $DB; + + $filecontent = 'example content'; + $contenthash = sha1($filecontent); + + $DB = $this->getMockBuilder(\moodle_database::class) + ->setMethods(['record_exists']) + ->getMockForAbstractClass(); + $DB->method('record_exists')->willReturn(true); + + $method = new ReflectionMethod(file_system::class, 'is_file_removable'); + $method->setAccessible(true); + $result = $method->invokeArgs(null, [$contenthash]); + + $this->assertFalse($result); + } + + /** + * Ensure that is_file_removable returns false if the file is not in use. + */ + public function test_is_file_removable_not_in_use() { + $this->resetAfterTest(); + global $DB; + + $filecontent = 'example content'; + $contenthash = sha1($filecontent); + + $DB = $this->getMockBuilder(\moodle_database::class) + ->setMethods(['record_exists']) + ->getMockForAbstractClass(); + $DB->method('record_exists')->willReturn(false); + + $method = new ReflectionMethod(file_system::class, 'is_file_removable'); + $method->setAccessible(true); + $result = $method->invokeArgs(null, [$contenthash]); + + $this->assertTrue($result); + } + + /** + * Test the stock implementation of get_content. + */ + public function test_get_content() { + global $CFG; + + // Mock the filesystem. + $filecontent = 'example content'; + $vfileroot = $this->setup_vfile_root(['sourcefile' => $filecontent]); + $filepath = \org\bovigo\vfs\vfsStream::url('root/sourcefile'); + + $file = $this->get_stored_file($filecontent); + + // Mock the file_system class. + // We need to override the get_remote_path_from_storedfile function. + $fs = $this->get_testable_mock(['get_remote_path_from_storedfile']); + $fs->method('get_remote_path_from_storedfile')->willReturn($filepath); + + $result = $fs->get_content($file); + + $this->assertEquals($filecontent, $result); + } + + /** + * Test the stock implementation of get_content. + */ + public function test_get_content_empty() { + global $CFG; + + $filecontent = ''; + $file = $this->get_stored_file($filecontent); + + // Mock the file_system class. + // We need to override the get_remote_path_from_storedfile function. + $fs = $this->get_testable_mock(['get_remote_path_from_storedfile']); + $fs->expects($this->never()) + ->method('get_remote_path_from_storedfile'); + + $result = $fs->get_content($file); + + $this->assertEquals($filecontent, $result); + } + + /** + * Ensure that the list_files function requires a local copy of the + * file, and passes the path to the packer. + */ + public function test_list_files() { + $filecontent = 'example content'; + $file = $this->get_stored_file($filecontent); + $filepath = __FILE__; + $expectedresult = (object) []; + + // Mock the file_system class. + $fs = $this->get_testable_mock(['get_local_path_from_storedfile']); + $fs->method('get_local_path_from_storedfile') + ->with($this->equalTo($file), $this->equalTo(true)) + ->willReturn(__FILE__); + + $packer = $this->getMockBuilder(file_packer::class) + ->setMethods(['list_files']) + ->getMockForAbstractClass(); + + $packer->expects($this->once()) + ->method('list_files') + ->with($this->equalTo($filepath)) + ->willReturn($expectedresult); + + $result = $fs->list_files($file, $packer); + + $this->assertEquals($expectedresult, $result); + } + + /** + * Ensure that the extract_to_pathname function requires a local copy of the + * file, and passes the path to the packer. + */ + public function test_extract_to_pathname() { + $filecontent = 'example content'; + $file = $this->get_stored_file($filecontent); + $filepath = __FILE__; + $expectedresult = (object) []; + $outputpath = '/path/to/output'; + + // Mock the file_system class. + $fs = $this->get_testable_mock(['get_local_path_from_storedfile']); + $fs->method('get_local_path_from_storedfile') + ->with($this->equalTo($file), $this->equalTo(true)) + ->willReturn(__FILE__); + + $packer = $this->getMockBuilder(file_packer::class) + ->setMethods(['extract_to_pathname']) + ->getMockForAbstractClass(); + + $packer->expects($this->once()) + ->method('extract_to_pathname') + ->with($this->equalTo($filepath), $this->equalTo($outputpath), $this->equalTo(null), $this->equalTo(null)) + ->willReturn($expectedresult); + + $result = $fs->extract_to_pathname($file, $packer, $outputpath); + + $this->assertEquals($expectedresult, $result); + } + + /** + * Ensure that the extract_to_storage function requires a local copy of the + * file, and passes the path to the packer. + */ + public function test_extract_to_storage() { + $filecontent = 'example content'; + $file = $this->get_stored_file($filecontent); + $filepath = __FILE__; + $expectedresult = (object) []; + $outputpath = '/path/to/output'; + + // Mock the file_system class. + $fs = $this->get_testable_mock(['get_local_path_from_storedfile']); + $fs->method('get_local_path_from_storedfile') + ->with($this->equalTo($file), $this->equalTo(true)) + ->willReturn(__FILE__); + + $packer = $this->getMockBuilder(file_packer::class) + ->setMethods(['extract_to_storage']) + ->getMockForAbstractClass(); + + $packer->expects($this->once()) + ->method('extract_to_storage') + ->with( + $this->equalTo($filepath), + $this->equalTo(42), + $this->equalTo('component'), + $this->equalTo('filearea'), + $this->equalTo('itemid'), + $this->equalTo('pathbase'), + $this->equalTo('userid'), + $this->equalTo(null) + ) + ->willReturn($expectedresult); + + $result = $fs->extract_to_storage($file, $packer, 42, 'component','filearea', 'itemid', 'pathbase', 'userid'); + + $this->assertEquals($expectedresult, $result); + } + + /** + * Ensure that the add_storedfile_to_archive function requires a local copy of the + * file, and passes the path to the archive. + */ + public function test_add_storedfile_to_archive_directory() { + $file = $this->get_stored_file('', '.'); + $archivepath = 'example'; + $expectedresult = (object) []; + + // Mock the file_system class. + $fs = $this->get_testable_mock(['get_local_path_from_storedfile']); + $fs->method('get_local_path_from_storedfile') + ->with($this->equalTo($file), $this->equalTo(true)) + ->willReturn(__FILE__); + + $archive = $this->getMockBuilder(file_archive::class) + ->setMethods([ + 'add_directory', + 'add_file_from_pathname', + ]) + ->getMockForAbstractClass(); + + $archive->expects($this->once()) + ->method('add_directory') + ->with($this->equalTo($archivepath)) + ->willReturn($expectedresult); + + $archive->expects($this->never()) + ->method('add_file_from_pathname'); + + $result = $fs->add_storedfile_to_archive($file, $archive, $archivepath); + + $this->assertEquals($expectedresult, $result); + } + + /** + * Ensure that the add_storedfile_to_archive function requires a local copy of the + * file, and passes the path to the archive. + */ + public function test_add_storedfile_to_archive_file() { + $file = $this->get_stored_file('example content'); + $filepath = __LINE__; + $archivepath = 'example'; + $expectedresult = (object) []; + + // Mock the file_system class. + $fs = $this->get_testable_mock(['get_local_path_from_storedfile']); + $fs->method('get_local_path_from_storedfile') + ->with($this->equalTo($file), $this->equalTo(true)) + ->willReturn($filepath); + + $archive = $this->getMockBuilder(file_archive::class) + ->setMethods([ + 'add_directory', + 'add_file_from_pathname', + ]) + ->getMockForAbstractClass(); + + $archive->expects($this->never()) + ->method('add_directory'); + + $archive->expects($this->once()) + ->method('add_file_from_pathname') + ->with( + $this->equalTo($archivepath), + $this->equalTo($filepath) + ) + ->willReturn($expectedresult); + + $result = $fs->add_storedfile_to_archive($file, $archive, $archivepath); + + $this->assertEquals($expectedresult, $result); + } + + /** + * Ensure that the add_to_curl_request function requires a local copy of the + * file, and passes the path to curl_file_create. + */ + public function test_add_to_curl_request() { + $file = $this->get_stored_file('example content'); + $filepath = __FILE__; + $archivepath = 'example'; + $key = 'myfile'; + + // Mock the file_system class. + $fs = $this->get_testable_mock(['get_local_path_from_storedfile']); + $fs->method('get_local_path_from_storedfile') + ->with($this->equalTo($file), $this->equalTo(true)) + ->willReturn($filepath); + + $request = (object) ['_tmp_file_post_params' => []]; + $fs->add_to_curl_request($file, $request, $key); + $this->assertArrayHasKey($key, $request->_tmp_file_post_params); + $this->assertEquals($filepath, $request->_tmp_file_post_params[$key]->name); + } + + /** + * Ensure that test_get_imageinfo_not_image returns false if the file + * passed was deemed to not be an image. + */ + public function test_get_imageinfo_not_image() { + $filecontent = 'example content'; + $file = $this->get_stored_file($filecontent); + + $fs = $this->get_testable_mock([ + 'is_image_from_storedfile', + ]); + + $fs->expects($this->once()) + ->method('is_image_from_storedfile') + ->with($this->equalTo($file)) + ->willReturn(false); + + $this->assertFalse($fs->get_imageinfo($file)); + } + + /** + * Ensure that test_get_imageinfo_not_image returns imageinfo. + */ + public function test_get_imageinfo() { + $filepath = '/path/to/file'; + $filecontent = 'example content'; + $expectedresult = (object) []; + $file = $this->get_stored_file($filecontent); + + $fs = $this->get_testable_mock([ + 'is_image_from_storedfile', + 'get_local_path_from_storedfile', + 'get_imageinfo_from_path', + ]); + + $fs->expects($this->once()) + ->method('is_image_from_storedfile') + ->with($this->equalTo($file)) + ->willReturn(true); + + $fs->expects($this->once()) + ->method('get_local_path_from_storedfile') + ->with($this->equalTo($file), $this->equalTo(true)) + ->willReturn($filepath); + + $fs->expects($this->once()) + ->method('get_imageinfo_from_path') + ->with($this->equalTo($filepath)) + ->willReturn($expectedresult); + + $this->assertEquals($expectedresult, $fs->get_imageinfo($file)); + } + + /** + * Ensure that is_image_from_storedfile always returns false for an + * empty file size. + */ + public function test_is_image_empty_filesize() { + $filecontent = 'example content'; + $file = $this->get_stored_file($filecontent, null, ['get_filesize']); + + $file->expects($this->once()) + ->method('get_filesize') + ->willReturn(0); + + $fs = $this->get_testable_mock(); + $this->assertFalse($fs->is_image_from_storedfile($file)); + } + + /** + * Ensure that is_image_from_storedfile behaves correctly based on + * mimetype. + * + * @dataProvider is_image_from_storedfile_provider + * @param string $mimetype Mimetype to test + * @param bool $isimage Whether this mimetype should be detected as an image + */ + public function test_is_image_from_storedfile_mimetype($mimetype, $isimage) { + $filecontent = 'example content'; + $file = $this->get_stored_file($filecontent, null, ['get_mimetype']); + + $file->expects($this->once()) + ->method('get_mimetype') + ->willReturn($mimetype); + + $fs = $this->get_testable_mock(); + $this->assertEquals($isimage, $fs->is_image_from_storedfile($file)); + } + + /** + * Test that get_imageinfo_from_path returns an appropriate response + * for an image. + */ + public function test_get_imageinfo_from_path() { + $filepath = __DIR__ . DIRECTORY_SEPARATOR . 'fixtures' . DIRECTORY_SEPARATOR . 'testimage.jpg'; + + // Get the filesystem mock. + $fs = $this->get_testable_mock(); + + $method = new ReflectionMethod(file_system::class, 'get_imageinfo_from_path'); + $method->setAccessible(true); + $result = $method->invokeArgs($fs, [$filepath]); + + $this->assertArrayHasKey('width', $result); + $this->assertArrayHasKey('height', $result); + $this->assertArrayHasKey('mimetype', $result); + $this->assertEquals('image/jpeg', $result['mimetype']); + } + + /** + * Test that get_imageinfo_from_path returns an appropriate response + * for a file which is not an image. + */ + public function test_get_imageinfo_from_path_no_image() { + $filepath = __FILE__; + + // Get the filesystem mock. + $fs = $this->get_testable_mock(); + + $method = new ReflectionMethod(file_system::class, 'get_imageinfo_from_path'); + $method->setAccessible(true); + $result = $method->invokeArgs($fs, [$filepath]); + + $this->assertFalse($result); + } + + /** + * Ensure that get_content_file_handle returns a valid file handle. + */ + public function test_get_content_file_handle_default() { + $filecontent = 'example content'; + $file = $this->get_stored_file($filecontent); + + $fs = $this->get_testable_mock(['get_remote_path_from_storedfile']); + $fs->method('get_remote_path_from_storedfile') + ->willReturn(__FILE__); + + // Note: We are unable to determine the mode in which the $fh was opened. + $fh = $fs->get_content_file_handle($file); + $this->assertTrue(is_resource($fh)); + $this->assertEquals('stream', get_resource_type($fh)); + fclose($fh); + } + + /** + * Ensure that get_content_file_handle returns a valid file handle for a gz file. + */ + public function test_get_content_file_handle_gz() { + $filecontent = 'example content'; + $file = $this->get_stored_file($filecontent); + + $fs = $this->get_testable_mock(['get_remote_path_from_storedfile']); + $fs->method('get_remote_path_from_storedfile') + ->willReturn(__DIR__ . DIRECTORY_SEPARATOR . 'fixtures' . DIRECTORY_SEPARATOR . 'test.tgz'); + + // Note: We are unable to determine the mode in which the $fh was opened. + $fh = $fs->get_content_file_handle($file, stored_file::FILE_HANDLE_GZOPEN); + $this->assertTrue(is_resource($fh)); + gzclose($fh); + } + + /** + * Ensure that get_content_file_handle returns an exception when calling for a invalid file handle type. + */ + public function test_get_content_file_handle_invalid() { + $filecontent = 'example content'; + $file = $this->get_stored_file($filecontent); + + $fs = $this->get_testable_mock(['get_remote_path_from_storedfile']); + $fs->method('get_remote_path_from_storedfile') + ->willReturn(__FILE__); + + $this->expectException('coding_exception', 'Unexpected file handle type'); + $fs->get_content_file_handle($file, -1); + } + + /** + * Test that mimetype_from_hash returns the correct mimetype with + * a file whose filename suggests mimetype. + */ + public function test_mimetype_from_hash_using_filename() { + $filepath = '/path/to/file/not/currently/on/disk'; + $filecontent = 'example content'; + $filename = 'test.jpg'; + $contenthash = sha1($filecontent); + + $fs = $this->get_testable_mock(['get_remote_path_from_hash']); + $fs->method('get_remote_path_from_hash')->willReturn($filepath); + + $result = $fs->mimetype_from_hash($contenthash, $filename); + $this->assertEquals('image/jpeg', $result); + } + + /** + * Test that mimetype_from_hash returns the correct mimetype with + * a locally available file whose filename does not suggest mimetype. + */ + public function test_mimetype_from_hash_using_file_content() { + $filepath = '/path/to/file/not/currently/on/disk'; + $filecontent = 'example content'; + $contenthash = sha1($filecontent); + $filename = 'example'; + + $filepath = __DIR__ . DIRECTORY_SEPARATOR . 'fixtures' . DIRECTORY_SEPARATOR . 'testimage.jpg'; + $fs = $this->get_testable_mock(['get_remote_path_from_hash']); + $fs->method('get_remote_path_from_hash')->willReturn($filepath); + + $result = $fs->mimetype_from_hash($contenthash, $filename); + $this->assertEquals('image/jpeg', $result); + } + + /** + * Test that mimetype_from_hash returns the correct mimetype with + * a remotely available file whose filename does not suggest mimetype. + */ + public function test_mimetype_from_hash_using_file_content_remote() { + $filepath = '/path/to/file/not/currently/on/disk'; + $filecontent = 'example content'; + $contenthash = sha1($filecontent); + $filename = 'example'; + + $filepath = __DIR__ . DIRECTORY_SEPARATOR . 'fixtures' . DIRECTORY_SEPARATOR . 'testimage.jpg'; + + $fs = $this->get_testable_mock([ + 'get_remote_path_from_hash', + 'is_file_readable_locally_by_hash', + 'get_local_path_from_hash', + ]); + + $fs->method('get_remote_path_from_hash')->willReturn('/path/to/remote/file'); + $fs->method('is_file_readable_locally_by_hash')->willReturn(false); + $fs->method('get_local_path_from_hash')->willReturn($filepath); + + $result = $fs->mimetype_from_hash($contenthash, $filename); + $this->assertEquals('image/jpeg', $result); + } + + /** + * Test that mimetype_from_storedfile returns the correct mimetype with + * a file whose filename suggests mimetype. + */ + public function test_mimetype_from_storedfile_empty() { + $file = $this->get_stored_file(''); + + $fs = $this->get_testable_mock(); + $result = $fs->mimetype_from_storedfile($file); + $this->assertNull($result); + } + + /** + * Test that mimetype_from_storedfile returns the correct mimetype with + * a file whose filename suggests mimetype. + */ + public function test_mimetype_from_storedfile_using_filename() { + $filepath = '/path/to/file/not/currently/on/disk'; + $fs = $this->get_testable_mock(['get_remote_path_from_storedfile']); + $fs->method('get_remote_path_from_storedfile')->willReturn($filepath); + + $file = $this->get_stored_file('example content', 'test.jpg'); + + $result = $fs->mimetype_from_storedfile($file); + $this->assertEquals('image/jpeg', $result); + } + + /** + * Test that mimetype_from_storedfile returns the correct mimetype with + * a locally available file whose filename does not suggest mimetype. + */ + public function test_mimetype_from_storedfile_using_file_content() { + $filepath = __DIR__ . DIRECTORY_SEPARATOR . 'fixtures' . DIRECTORY_SEPARATOR . 'testimage.jpg'; + $fs = $this->get_testable_mock(['get_remote_path_from_storedfile']); + $fs->method('get_remote_path_from_storedfile')->willReturn($filepath); + + $file = $this->get_stored_file('example content'); + + $result = $fs->mimetype_from_storedfile($file); + $this->assertEquals('image/jpeg', $result); + } + + /** + * Test that mimetype_from_storedfile returns the correct mimetype with + * a remotely available file whose filename does not suggest mimetype. + */ + public function test_mimetype_from_storedfile_using_file_content_remote() { + $filepath = __DIR__ . DIRECTORY_SEPARATOR . 'fixtures' . DIRECTORY_SEPARATOR . 'testimage.jpg'; + + $fs = $this->get_testable_mock([ + 'get_remote_path_from_storedfile', + 'is_file_readable_locally_by_storedfile', + 'get_local_path_from_storedfile', + ]); + + $fs->method('get_remote_path_from_storedfile')->willReturn('/path/to/remote/file'); + $fs->method('is_file_readable_locally_by_storedfile')->willReturn(false); + $fs->method('get_local_path_from_storedfile')->willReturn($filepath); + + $file = $this->get_stored_file('example content'); + + $result = $fs->mimetype_from_storedfile($file); + $this->assertEquals('image/jpeg', $result); + } + + /** + * Data Provider for is_image_from_storedfile tests. + * + * @return array + */ + public function is_image_from_storedfile_provider() { + return array( + 'Standard image' => array('image/png', true), + 'Made up document/image' => array('document/image', false), + ); + } + + /** + * Data provider for get_local_path_from_storedfile tests. + * + * @return array + */ + public function get_local_path_from_storedfile_provider() { + return [ + 'default args (nofetch)' => [ + 'args' => [], + 'fetch' => 0, + ], + 'explicit: nofetch' => [ + 'args' => [false], + 'fetch' => 0, + ], + 'explicit: fetch' => [ + 'args' => [true], + 'fetch' => 1, + ], + ]; + } +} diff --git a/lib/filestorage/tests/fixtures/test.tgz b/lib/filestorage/tests/fixtures/test.tgz new file mode 100644 index 0000000000000..6c92dce1e6034 Binary files /dev/null and b/lib/filestorage/tests/fixtures/test.tgz differ diff --git a/lib/moodlelib.php b/lib/moodlelib.php index 966112fa5637b..497fea4a837c5 100644 --- a/lib/moodlelib.php +++ b/lib/moodlelib.php @@ -6369,30 +6369,23 @@ function email_is_not_allowed($email) { * * @return file_storage */ -function get_file_storage() { +function get_file_storage($reset = false) { global $CFG; static $fs = null; + if ($reset) { + $fs = null; + return; + } + if ($fs) { return $fs; } require_once("$CFG->libdir/filelib.php"); - if (isset($CFG->filedir)) { - $filedir = $CFG->filedir; - } else { - $filedir = $CFG->dataroot.'/filedir'; - } - - if (isset($CFG->trashdir)) { - $trashdirdir = $CFG->trashdir; - } else { - $trashdirdir = $CFG->dataroot.'/trashdir'; - } - - $fs = new file_storage($filedir, $trashdirdir, "$CFG->tempdir/filestorage", $CFG->directorypermissions, $CFG->filepermissions); + $fs = new file_storage(); return $fs; } diff --git a/lib/upgrade.txt b/lib/upgrade.txt index be273d5cddc3b..6b5d4acbd2fbc 100644 --- a/lib/upgrade.txt +++ b/lib/upgrade.txt @@ -18,6 +18,10 @@ information provided here is intended especially for developers. * The mcore YUI rollup which included various YUI modules such as moodle-core-notification is no longer included on every page. Missing YUI depdencies may be exposed by this change (e.g. missing a requirement on moodle-core-notification when using M.core.dialogue). +* The following functions have been deprecated and should not be used any more: + - file_storage::try_content_recovery - See MDL-46375 for more information + - file_storage::content_exists - See MDL-46375 for more information + - file_storage::deleted_file_cleanup - See MDL-46375 for more information === 3.2 === diff --git a/question/format/blackboard_six/formatbase.php b/question/format/blackboard_six/formatbase.php index 5033827650a9d..a2bf4fbff8b43 100644 --- a/question/format/blackboard_six/formatbase.php +++ b/question/format/blackboard_six/formatbase.php @@ -47,7 +47,7 @@ public function provide_import() { /** * Check if the given file is capable of being imported by this plugin. - * As {@link file_storage::mimetype()} now uses finfo PHP extension if available, + * As {@link file_storage::mimetype()} may use finfo PHP extension if available, * the value returned by $file->get_mimetype for a .dat file is not the same on all servers. * So we must made 2 checks to verify if the plugin can import the file. * @param stored_file $file the file to check diff --git a/repository/lib.php b/repository/lib.php index 5d0d905be5b8a..5cfd4bf6e1a4e 100644 --- a/repository/lib.php +++ b/repository/lib.php @@ -1717,12 +1717,13 @@ public function import_external_file_contents(stored_file $file, $maxbytes = 0) 'size' => $maxbytesdisplay)); } $fs = get_file_storage(); - $contentexists = $fs->content_exists($file->get_contenthash()); - if ($contentexists && $file->get_filesize() && $file->get_contenthash() === sha1('')) { - // even when 'file_storage::content_exists()' returns true this may be an empty - // content for the file that was not actually downloaded - $contentexists = false; - } + + // If a file has been downloaded, the file record should report both a positive file + // size, and a contenthash which does not related to empty content. + // If thereis no file size, or the contenthash is for an empty file, then the file has + // yet to be successfully downloaded. + $contentexists = $file->get_filesize() && $file->get_contenthash() !== sha1(''); + if (!$file->get_status() && $contentexists) { // we already have the content in moodle filepool and it was synchronised recently. // Repositories may overwrite it if they want to force synchronisation anyway!