Add data prep project

ingjjaa2 · Aug 24, 2019 · c2d8f54 · c2d8f54
1 parent 938f2dd
commit c2d8f54
Show file tree

Hide file tree

Showing 7 changed files with 20,828 additions and 2 deletions.
diff --git a/MLNetExamples/FeatureImportance/Program.cs b/MLNetExamples/FeatureImportance/Program.cs
@@ -24,8 +24,57 @@ static void Main(string[] args)
                 .Append(context.Regression.Trainers.LbfgsPoissonRegression());
 
             var model = pipeline.Fit(data);
+            var transformedData = model.Transform(data);
 
-            //context.
+            // Get weights of model
+            var linearModel = model.LastTransformer.Model;
+
+            var weights = linearModel.Weights;
+
+            Console.WriteLine("Weights:");
+            var weightsResult = "";
+
+            foreach (var weight in weights)
+            {
+                weightsResult += $"{weight} ";
+            }
+
+            Console.WriteLine(weightsResult);
+            Console.WriteLine(Environment.NewLine);
+
+            // Get global feature importance
+            var lastTransformer = model.LastTransformer;
+
+            var featureImportance = context.Regression.PermutationFeatureImportance(lastTransformer, transformedData);
+
+            Console.WriteLine("Global feature importance:");
+            for (int i = 0; i < featureImportance.Count(); i++)
+            {
+                Console.WriteLine($"Feature - {features[i]}: Difference in RMS - {featureImportance[i].RootMeanSquaredError.Mean}");
+            }
+
+            Console.WriteLine(Environment.NewLine);
+
+            // Get feature importance for each row
+            var featureContribution = context.Transforms.CalculateFeatureContribution(lastTransformer, normalize: false);
+
+            var featureContributionResults = featureContribution.Fit(transformedData).Transform(transformedData);
+
+            var topTenRows = context.Data.TakeRows(featureContributionResults, 10);
+
+            var scoringEnumerator = context.Data.CreateEnumerable<HousingData>(topTenRows, reuseRowObject: true);
+
+            Console.WriteLine("Row feature importance:");
+            Console.WriteLine("Households - Housing Median Age - Median Income - Ocean Proximity");
+            var globalResults = "";
+            foreach (var row in scoringEnumerator)
+            {
+                globalResults += $"{row.Households} - {row.HousingMedianAge} = {row.MedianIncome} - {row.OceanProximity}\n";
+            }
+
+            Console.WriteLine(globalResults);
+
+            Console.ReadLine();
         }
     }
 }
diff --git a/MLNetExamples/MLNetExamples.sln b/MLNetExamples/MLNetExamples.sln
@@ -29,7 +29,9 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DatabaseLoader", "DatabaseL
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AzureFunction", "AzureFunction\AzureFunction.csproj", "{0E54A183-8D61-4BA6-A6EA-BF24D55661DC}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "FeatureImportance", "FeatureImportance\FeatureImportance.csproj", "{D8DF42E5-501B-458F-9289-813875217AD4}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FeatureImportance", "FeatureImportance\FeatureImportance.csproj", "{D8DF42E5-501B-458F-9289-813875217AD4}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SelectAndShuffle", "SelectAndShuffle\SelectAndShuffle.csproj", "{414D7AD9-C9F9-4485-AC5B-15FB8BDF416B}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
@@ -93,6 +95,10 @@ Global
 		{D8DF42E5-501B-458F-9289-813875217AD4}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{D8DF42E5-501B-458F-9289-813875217AD4}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{D8DF42E5-501B-458F-9289-813875217AD4}.Release|Any CPU.Build.0 = Release|Any CPU
+		{414D7AD9-C9F9-4485-AC5B-15FB8BDF416B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{414D7AD9-C9F9-4485-AC5B-15FB8BDF416B}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{414D7AD9-C9F9-4485-AC5B-15FB8BDF416B}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{414D7AD9-C9F9-4485-AC5B-15FB8BDF416B}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

diff --git a/MLNetExamples/SelectAndShuffle/HousingData.cs b/MLNetExamples/SelectAndShuffle/HousingData.cs
@@ -0,0 +1,37 @@
+using Microsoft.ML.Data;
+
+namespace SelectAndShuffle
+{
+    public class HousingData
+    {
+        [LoadColumn(0)]
+        public float Longitude { get; set; }
+
+        [LoadColumn(1)]
+        public float Latitude { get; set; }
+
+        [LoadColumn(2)]
+        public float HousingMedianAge { get; set; }
+
+        [LoadColumn(3)]
+        public float TotalRooms { get; set; }
+
+        [LoadColumn(4)]
+        public float TotalBedrooms { get; set; }
+
+        [LoadColumn(5)]
+        public float Population { get; set; }
+
+        [LoadColumn(6)]
+        public float Households { get; set; }
+
+        [LoadColumn(7)]
+        public float MedianIncome { get; set; }
+
+        [LoadColumn(8), ColumnName("Label")]
+        public float MedianHouseValue { get; set; }
+
+        [LoadColumn(9)]
+        public string OceanProximity{ get; set; }
+    }
+}
diff --git a/MLNetExamples/SelectAndShuffle/HousingPrediction.cs b/MLNetExamples/SelectAndShuffle/HousingPrediction.cs
@@ -0,0 +1,10 @@
+using Microsoft.ML.Data;
+
+namespace SelectAndShuffle
+{
+    public class HousingPrediction
+    {
+        [ColumnName("Score")]
+        public float PredictedPrice { get; set; }
+    }
+}
diff --git a/MLNetExamples/SelectAndShuffle/Program.cs b/MLNetExamples/SelectAndShuffle/Program.cs
@@ -0,0 +1,65 @@
+using Microsoft.ML;
+using System;
+
+namespace SelectAndShuffle
+{
+    class Program
+    {
+        static void Main(string[] args)
+        {
+            var context = new MLContext();
+
+            var data = context.Data.LoadFromTextFile<HousingData>("./housing.csv", hasHeader: true, separatorChar: ',');
+
+            // Select columns
+            var selectCols = context.Transforms.SelectColumns("HousingMedianAge", "TotalBedrooms");
+            var selectColsTransform = selectCols.Fit(data).Transform(data);
+
+            //DisplayColumns(selectColsTransform);
+
+            // Drop columns
+            var dropCols = context.Transforms.DropColumns("Latitude", "Longitude");
+            var dropColsTransform = dropCols.Fit(data).Transform(data);
+
+            //DisplayColumns(dropColsTransform);
+
+            // Shuffle rows
+            //DisplayColumns(data);
+
+            var shuffleRows = context.Data.ShuffleRows(data, seed: 42);
+
+            //DisplayColumns(shuffleRows);
+
+            // Take rows
+            var takeRows = context.Data.TakeRows(data, 2);
+
+            //DisplayColumns(takeRows);
+
+            // Filter rows
+            var filterRows = context.Data.FilterRowsByColumn(data, "Population", lowerBound: 0, upperBound: 1000);
+
+            DisplayColumns(filterRows);
+
+            Console.ReadLine();
+        }
+
+        private static void DisplayColumns(IDataView data)
+        {
+            var preview = data.Preview(maxRows: 5);
+
+            string previewData = "";
+
+            for (int i = 0; i < preview.RowView.Length; i++)
+            {
+                foreach (var item in preview.RowView[i].Values)
+                {
+                    previewData += $"{item.Key}: {item.Value} ";
+                }
+
+                Console.WriteLine("----------------------------------");
+                Console.WriteLine(previewData);
+                previewData = "";
+            }
+        }
+    }
+}
diff --git a/MLNetExamples/SelectAndShuffle/SelectAndShuffle.csproj b/MLNetExamples/SelectAndShuffle/SelectAndShuffle.csproj
@@ -0,0 +1,18 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>netcoreapp2.2</TargetFramework>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.ML" Version="1.3.1" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <None Update="housing.csv">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+  </ItemGroup>
+
+</Project>