Refactor object store file handling (QuantConnect#7438)

- Add FileHandler for local object store file operations

Engine/Storage/FileHandler.cs

@@ -0,0 +1,98 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System.IO;
+using System.Collections.Generic;
+
+namespace QuantConnect.Lean.Engine.Storage
+{
+    /// <summary>
+    /// Raw file handler
+    /// </summary>
+    /// <remarks>Useful to abstract file operations for <see cref="LocalObjectStore"/></remarks>
+    public class FileHandler
+    {
+        /// <summary>
+        /// True if the given file path exists
+        /// </summary>
+        public virtual bool Exists(string path)
+        {
+            return File.Exists(path);
+        }
+
+        /// <summary>
+        /// Will delete the given file path
+        /// </summary>
+        public virtual void Delete(string path)
+        {
+            File.Delete(path);
+        }
+
+        /// <summary>
+        /// Will write the given byte array at the target file path
+        /// </summary>
+        public virtual void WriteAllBytes(string path, byte[] data)
+        {
+            File.WriteAllBytes(path, data);
+        }
+
+        /// <summary>
+        /// Read all bytes in the given file path
+        /// </summary>
+        public virtual byte[] ReadAllBytes(string path)
+        {
+            return File.ReadAllBytes(path);
+        }
+
+        /// <summary>
+        /// Will try to fetch the given file length, will return 0 if it doesn't exist
+        /// </summary>
+        public virtual long TryGetFileLength(string path)
+        {
+            var fileInfo = new FileInfo(path);
+            if (fileInfo.Exists)
+            {
+                return fileInfo.Length;
+            }
+            return 0;
+        }
+
+        /// <summary>
+        /// True if the given directory exists
+        /// </summary>
+        public virtual bool DirectoryExists(string path)
+        {
+            return Directory.Exists(path);
+        }
+
+        /// <summary>
+        /// Create the requested directory path
+        /// </summary>
+        public virtual DirectoryInfo CreateDirectory(string path)
+        {
+            return Directory.CreateDirectory(path);
+        }
+
+        /// <summary>
+        /// Enumerate the files in the target path
+        /// </summary>
+        public virtual IEnumerable<FileInfo> EnumerateFiles(string path, string pattern, SearchOption searchOption, out string rootfolder)
+        {
+            var directoryInfo = new DirectoryInfo(path);
+            rootfolder = directoryInfo.FullName;
+            return directoryInfo.EnumerateFiles(pattern, searchOption);
+        }
+    }
+}

Engine/Storage/LocalObjectStore.cs

@@ -77,6 +77,11 @@ public class LocalObjectStore : IObjectStore
         /// </summary>
         protected string AlgorithmStorageRoot { get; private set; }
 
+        /// <summary>
+        /// The file handler instance to use
+        /// </summary>
+        protected FileHandler FileHandler { get; set; } = new ();
+
         /// <summary>
         /// Initializes the object store
         /// </summary>
@@ -86,11 +91,10 @@ public class LocalObjectStore : IObjectStore
         /// <param name="controls">The job controls instance</param>
         public virtual void Initialize(int userId, int projectId, string userToken, Controls controls)
         {
-            // absolute path including algorithm name
             AlgorithmStorageRoot = _storageRoot;
 
             // create the root path if it does not exist
-            Directory.CreateDirectory(AlgorithmStorageRoot);
+            var directoryInfo = FileHandler.CreateDirectory(AlgorithmStorageRoot);
 
             Controls = controls;
 
@@ -100,7 +104,7 @@ public virtual void Initialize(int userId, int projectId, string userToken, Cont
                 _persistenceTimer = new Timer(_ => Persist(), null, Controls.PersistenceIntervalSeconds * 1000, Timeout.Infinite);
             }
 
-            Log.Trace($"LocalObjectStore.Initialize(): Storage Root: {new FileInfo(AlgorithmStorageRoot).FullName}. StorageFileCount {controls.StorageFileCount}. StorageLimit {BytesToMb(controls.StorageLimit)}MB");
+            Log.Trace($"LocalObjectStore.Initialize(): Storage Root: {directoryInfo.FullName}. StorageFileCount {controls.StorageFileCount}. StorageLimit {BytesToMb(controls.StorageLimit)}MB");
         }
 
         /// <summary>
@@ -122,10 +126,9 @@ private IEnumerable<ObjectStoreEntry> GetObjectStoreEntries(bool loadContent, bo
                         }
                     }
 
-                    var rootFolder = new DirectoryInfo(AlgorithmStorageRoot);
-                    foreach (var file in rootFolder.EnumerateFiles("*", SearchOption.AllDirectories))
+                    foreach (var file in FileHandler.EnumerateFiles(AlgorithmStorageRoot, "*", SearchOption.AllDirectories, out var rootFolder))
                     {
-                        var path = NormalizePath(file.FullName.RemoveFromStart(rootFolder.FullName));
+                        var path = NormalizePath(file.FullName.RemoveFromStart(rootFolder));
 
                         ObjectStoreEntry objectStoreEntry;
                         if (loadContent)
@@ -198,7 +201,7 @@ public bool ContainsKey(string path)
 
             // if we don't have the file but it exists, be friendly and register it
             var filePath = PathForKey(path);
-            if (File.Exists(filePath))
+            if (FileHandler.Exists(filePath))
             {
                 _storage[path] = new ObjectStoreEntry(path, null);
                 return true;
@@ -321,11 +324,7 @@ protected virtual bool IsWithinStorageLimit(string path, byte[] contents, bool t
                     }
                     else
                     {
-                        var fileInfo = new FileInfo(PathForKey(kvp.Path));
-                        if (fileInfo.Exists)
-                        {
-                            expectedStorageSizeBytes += fileInfo.Length;
-                        }
+                        expectedStorageSizeBytes += FileHandler.TryGetFileLength(PathForKey(kvp.Path));
                     }
                 }
             }
@@ -372,9 +371,9 @@ public bool Delete(string path)
             var wasInCache = _storage.TryRemove(path, out var _);
 
             var filePath = PathForKey(path);
-            if (File.Exists(filePath))
+            if (FileHandler.Exists(filePath))
             {
-                File.Delete(filePath);
+                FileHandler.Delete(filePath);
                 return true;
             }
 
@@ -516,11 +515,11 @@ protected virtual bool PersistData()
                         var filePath = PathForKey(kvp.Key);
                         // directory might not exist for custom prefix
                         var parentDirectory = Path.GetDirectoryName(filePath);
-                        if (!Directory.Exists(parentDirectory))
+                        if (!FileHandler.DirectoryExists(parentDirectory))
                         {
-                            Directory.CreateDirectory(parentDirectory);
+                            FileHandler.CreateDirectory(parentDirectory);
                         }
-                        File.WriteAllBytes(filePath, kvp.Value.Data);
+                        FileHandler.WriteAllBytes(filePath, kvp.Value.Data);
 
                         // clear the dirty flag
                         kvp.Value.SetClean();
@@ -562,17 +561,17 @@ private static string NormalizePath(string path)
             return path.TrimStart('.').TrimStart('/', '\\').Replace('\\', '/');
         }
 
-        private static bool TryCreateObjectStoreEntry(string filePath, string path, out ObjectStoreEntry objectStoreEntry)
+        private bool TryCreateObjectStoreEntry(string filePath, string path, out ObjectStoreEntry objectStoreEntry)
         {
             var count = 0;
             do
             {
                 count++;
                 try
                 {
-                    if (File.Exists(filePath))
+                    if (FileHandler.Exists(filePath))
                     {
-                        objectStoreEntry = new ObjectStoreEntry(path, File.ReadAllBytes(filePath));
+                        objectStoreEntry = new ObjectStoreEntry(path, FileHandler.ReadAllBytes(filePath));
                         return true;
                     }
                     objectStoreEntry = null;