more fluent reader with options

Author: Ivan Gavryliuk

src/DataFrame.Formats/Csv/CsvFormatReader.cs

@@ -9,25 +9,20 @@
 
 namespace DataFrame.Formats.Csv
 {
-   class CsvFormatReader : IFormatReader
+   static class CsvFormatReader
    {
       private static readonly Dictionary<Type, Type> _inferredTypeToParquetType = new Dictionary<Type, Type>
       {
          { typeof(byte), typeof(int) }
       };
 
-      public Frame FromStream(Stream inputStream)
-      {
-         throw new NotImplementedException();
-      }
-
       /// <summary>
       /// Reads csv stream into dataset
       /// </summary>
       /// <param name="csvStream">CSV stream</param>
       /// <param name="options">Options for reader, optional</param>
       /// <returns>Correct dataset</returns>
-      public static Matrix<object> ReadToDataSet(Stream csvStream, CsvOptions options = null)
+      public static Frame ReadToFrame(Stream csvStream, CsvOptions options = null)
       {
          if (csvStream == null) throw new ArgumentNullException(nameof(csvStream));
 
@@ -75,49 +70,30 @@ public static Matrix<object> ReadToDataSet(Stream csvStream, CsvOptions options
             rowCount += 1;
          }
 
-         Matrix<object> result;
-
          //set schema
          if (options.InferSchema)
          {
-            result = InferSchema(headers, rowCount, columnValues);
-         }
-         else
-         {
-            var schema = headers.Select(h => new ColumnSchema<string>(h)).ToList();
-            result = new Matrix(schema, rowCount);
+            return InferSchema(headers, rowCount, columnValues);
          }
 
-         //assign values
-         foreach(KeyValuePair<int, IList> pair in columnValues)
-         {
-            int r = 0;
-            foreach(object value in pair.Value)
-            {
-               result[pair.Key, r++] = value;
-            }
-         }
-
-         return result;
+         return new Frame(headers.Select((name, i) => new Series<string>(name, (List<string>)columnValues[i])));
       }
 
 
-      private static Matrix InferSchema(string[] headers, int rowCount, Dictionary<int, IList> columnValues)
+      private static Frame InferSchema(string[] headers, int rowCount, IReadOnlyDictionary<int, IList> columnValues)
       {
-         var elements = new List<ColumnSchema>();
+         var series = new List<Series>();
          for (int i = 0; i < headers.Length; i++)
          {
             IList cv = columnValues[i];
             Type columnType = cv.Cast<string>().ToArray().InferType(out IList typedValues);
 
             Type ct;
             if (!_inferredTypeToParquetType.TryGetValue(columnType, out ct)) ct = columnType;
-            elements.Add(new ColumnSchema(headers[i], ct));
-
-            columnValues[i] = typedValues;
+            series.Add(new Series(ct, headers[i], typedValues));
          }
 
-         return new Matrix(elements, rowCount);
+         return new Frame(series);
       }
    }
 }

src/DataFrame.Formats/FluentExtensions.cs

@@ -1,13 +1,24 @@
-using DataFrame.Formats.Parquet;
+using System.IO;
+using DataFrame.Formats.Csv;
+using DataFrame.Formats.Parquet;
 using DataFrame.Math.Data;
+using Parquet;
+using Parquet.Data;
 
-namespace DataFrame.Formats
+namespace DataFrame.Math.Data
 {
    public static class FluentExtensions
    {
-      public static IFormatReader Parquet(this FluentReader entry)
+      public static Frame Parquet(this FluentReader reader, Stream inputStream, ParquetOptions parquetOptions = null, ReaderOptions readerOptions = null)
       {
-         return new ParquetFormatReader();
+         DataSet ds = ParquetReader.Read(inputStream, parquetOptions, readerOptions);
+
+         return ParquetConverter.ConvertFromParquet(ds);
+      }
+
+      public static Frame Csv(this FluentReader reader, Stream inputStream, CsvOptions options = null)
+      {
+         return CsvFormatReader.ReadToFrame(inputStream, options);
       }
    }
 }

src/DataFrame.Formats/IFormatReader.cs

@@ -35,5 +35,11 @@ public IReadOnlyCollection<object> GetRow(int i)
 
          return f;
       }
+
+      #region [ Fluent Extensions ]
+
+      public static FluentReader Read => new FluentReader();
+
+      #endregion
    }
 }

src/DataFrame.Formats/Parquet/ParquetFormatReader.cs

@@ -4,17 +4,9 @@
 
 namespace DataFrame.Math.Data
 {
-   public class Matrix : Matrix<object>
-   {
-      public Matrix(IReadOnlyCollection<ColumnSchema> schema, int rowCount) : base(schema.Count, rowCount)
-      {
-      }
-   }
-
    public class Matrix<T>
    {
       private readonly MatrixStorage<T> _storage;
-      private readonly List<ColumnSchema> _schema = new List<ColumnSchema>();
 
       public Matrix(int columns, int rows) : this(columns, rows, new SimpleMatrixStorage<T>(columns, rows))
       {
@@ -51,15 +43,6 @@ private void ValidateIndexInRange(int column, int row)
          //todo:
       }
 
-      #region [ Fluent Extensions ]
-
-      public static FluentReader Read()
-      {
-         return new FluentReader();
-      }
-
-      #endregion
-
       public override string ToString()
       {
          var sb = new StringBuilder();

src/DataFrame.Math/Data/ColumnSchema.cs

@@ -11,7 +11,7 @@ public void Read_parquet_file()
       {
          using (FileStream fs = File.OpenRead(GetDataFilePath("alltypes.snappy.parquet")))
          {
-            Frame frame = Matrix.Read().Parquet().FromStream(fs);
+            Frame frame = Frame.Read.Parquet(fs);
          }
       }