Memory-map file to speed up loading

Use Boost.Iostreams to stream a memory-mapped file instead of reading a
normal file stream's buffer into a string stream all at once, it speeds
up loading Skyrim.esm by ~ 40% (1.3s). It also reduces total memory
consumption over reading the whole file into a char buffer and parsing
the buffer, which is the other fast I/O option.

.travis.yml

@@ -11,6 +11,7 @@ addons:
       - ubuntu-toolchain-r-test
     packages:
       - libboost-filesystem1.55-dev
+      - libboost-iostreams1.55-dev
       - libboost-locale1.55-dev
       - gcc-5
       - g++-5

CMakeLists.txt

@@ -12,7 +12,7 @@ set (Boost_USE_MULTITHREADED ON)
 set (Boost_USE_STATIC_RUNTIME ${PROJECT_STATIC_RUNTIME})
 
 find_package(GTest REQUIRED)
-find_package(Boost REQUIRED COMPONENTS filesystem system locale)
+find_package(Boost REQUIRED COMPONENTS iostreams filesystem system locale)
 
 set(PROJECT_HEADERS "${CMAKE_SOURCE_DIR}/include/libespm/FormId.h"
                     "${CMAKE_SOURCE_DIR}/include/libespm/GameId.h"

README.md

@@ -16,7 +16,7 @@ Libespm focuses on providing a useful API to [libloadorder](https://github.com/W
 
 ## Requirements
 
-* [Boost](http://www.boost.org) Filesystem and Locale libraries (tested with v1.55.0 and v1.59.0)
+* [Boost](http://www.boost.org) Filesystem, Iostreams and Locale libraries (tested with v1.55.0 and v1.59.0)
 
 ## Build Instructions
 

include/libespm/Plugin.h

@@ -28,6 +28,8 @@
 #include <boost/filesystem.hpp>
 #include <boost/filesystem/fstream.hpp>
 #include <boost/locale/encoding.hpp>
+#include <boost/iostreams/device/mapped_file.hpp>
+#include <boost/iostreams/stream.hpp>
 
 #include "FormId.h"
 #include "Group.h"
@@ -47,7 +49,10 @@ namespace libespm {
     inline void load(const boost::filesystem::path& filepath, bool loadHeaderOnly = false) {
       name = filepath.filename().string();
 
-      boost::filesystem::ifstream input(filepath, std::ios::binary);
+      // Memory-map the file, this can save memory and significantly improve
+      // performance when reading large plugins.
+      boost::iostreams::mapped_file_source mmap(filepath);
+      boost::iostreams::stream<boost::iostreams::mapped_file_source> input(mmap, std::ios::binary);
       input.exceptions(std::ios_base::badbit | std::ios_base::failbit);
 
       headerRecord.read(input, gameId, false);
@@ -58,33 +63,22 @@ namespace libespm {
       if (loadHeaderOnly)
         return;
 
-      // Read the rest of the file in at once.
-      std::stringstream bufferStream;
-      bufferStream.exceptions(std::ios_base::badbit | std::ios_base::failbit);
-
-      // Jump back to the beginning of the file.
-      input.seekg(0, std::ios_base::beg);
-      bufferStream << input.rdbuf();
-
-      // Re-read the header record.
-      headerRecord.read(bufferStream, gameId, false);
-
       // If the filename ends in ".ghost", trim that extension.
       std::string trimmedName = trimToEspm(name);
 
       std::vector<std::string> masters = getMasters();
       uintmax_t fileSize = boost::filesystem::file_size(filepath);
       if (gameId == GameId::MORROWIND) {
-        while (bufferStream.good() && (uintmax_t)bufferStream.tellg() < fileSize) {
+        while (input.good() && (uintmax_t)input.tellg() < fileSize) {
           Record record;
-          record.read(bufferStream, gameId, false);
+          record.read(input, gameId, false);
           formIds.insert(FormId(trimmedName, masters, record.getFormId()));
         }
       }
       else {
-        while (bufferStream.good() && (uintmax_t)bufferStream.tellg() < fileSize) {
+        while (input.good() && (uintmax_t)input.tellg() < fileSize) {
           Group group;
-          group.read(bufferStream, gameId, true);
+          group.read(input, gameId, true);
           for (const auto& formId : group.getRecordFormIds()) {
             formIds.insert(FormId(trimmedName, masters, formId));
           }