First e2e done

FsDNN.DemoApp/FsDNN.DemoApp.fsproj

@@ -3,15 +3,21 @@
     <PropertyGroup>
         <OutputType>Exe</OutputType>
         <TargetFramework>netcoreapp2.1</TargetFramework>
+        <AssemblyName>KS.FsDNN.DemoApp</AssemblyName>
+        <RootNamespace>KS.FsDNN.DemoApp</RootNamespace>
     </PropertyGroup>
 
     <ItemGroup>
-        <Compile Include="Program.fs"/>
+        <Compile Include="Program.fs" />
     </ItemGroup>
 
     <ItemGroup>
         <PackageReference Update="FSharp.Core" Version="4.7.2" />
         <PackageReference Include="Microsoft.Data.Analysis" Version="0.4.0" />
     </ItemGroup>
 
+    <ItemGroup>
+      <ProjectReference Include="..\FsDNN\FsDNN.fsproj" />
+    </ItemGroup>
+
 </Project>

FsDNN.Tests/FsDNN.Tests.fsproj

@@ -5,16 +5,19 @@
 
         <IsPackable>false</IsPackable>
         <GenerateProgramFile>false</GenerateProgramFile>
+        <AssemblyName>KS.FsDNN.Tests</AssemblyName>
+        <RootNamespace>KS.FsDNN.Tests</RootNamespace>
     </PropertyGroup>
 
     <ItemGroup>
-        <Compile Include="Tests.fs"/>
-        <Compile Include="Program.fs"/>
+        <Compile Include="TestHelpers.fs" />
+        <Compile Include="NetTests.fs" />
+        <Compile Include="Program.fs" />
     </ItemGroup>
 
     <ItemGroup>
-        <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.5.0"/>
-        <PackageReference Include="coverlet.collector" Version="1.2.0"/>
+        <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.5.0" />
+        <PackageReference Include="coverlet.collector" Version="1.2.0" />
         <PackageReference Update="FSharp.Core" Version="4.7.2" />
     </ItemGroup>
 
@@ -31,4 +34,8 @@
         </PackageReference>
     </ItemGroup>
 
+    <ItemGroup>
+      <ProjectReference Include="..\FsDNN\FsDNN.fsproj" />
+    </ItemGroup>
+
 </Project>

FsDNN.Tests/NetTests.fs

@@ -0,0 +1,19 @@
+module KS.FsDNN.Tests.Net
+
+open KS.FsDNN
+open MathNet.Numerics.LinearAlgebra
+open Xunit
+
+[<Fact>]
+let ``Forward propagate - 1 hidden layer`` () =
+    let il = { nx = 2 }
+    let ll = SoftMax {| nc = 2 |}
+
+    let net = Net.makeLayers 0 il [] ll
+
+    let X = matrix [[ 0.5 ]; [ 1.3 ]]
+
+    let scores = Net.forward net X
+
+    let expected = [[ 0.10289726 ]; [ 0.89710273 ]]
+    scores |> shouldBeEquivalent expected

FsDNN.Tests/TestHelpers.fs

@@ -0,0 +1,27 @@
+namespace KS.FsDNN.Tests
+
+[<AutoOpen>]
+module TestHelpers =
+
+  open FluentAssertions
+  open FluentAssertions.Equivalency
+  open System
+  open MathNet.Numerics.LinearAlgebra
+
+  [<Literal>]
+  let precision = 5e-7;
+
+  let doubleComparisonOptions<'TExpectation> (o: EquivalencyAssertionOptions<'TExpectation>): EquivalencyAssertionOptions<'TExpectation> =
+    let action = fun (ctx: IAssertionContext<double>) -> ctx.Subject.Should().BeApproximately(ctx.Expectation, precision, String.Empty, Array.Empty<obj>()) |> ignore
+    o.Using<double>(Action<IAssertionContext<double>>(action)).WhenTypeIs<double>()
+
+  let shouldBeEquivalent (a: double list list) (m: Matrix<double>) =
+    m.ToArray().Should().BeEquivalentTo(array2D a, doubleComparisonOptions, String.Empty, Array.empty) |> ignore
+
+  let shouldBeEquivalentM2 (a: double [,]) (m: Matrix<double>) =
+    m.ToArray().Should().BeEquivalentTo(a, doubleComparisonOptions, String.Empty, Array.empty) |> ignore
+
+  let shouldBeApproximately (a1: double) (a2) =
+    a1.Should().BeApproximately(a2, precision, String.Empty, Array.empty) |> ignore
+
+  let toM (rs: double list list) = rs |> array2D |> CreateMatrix.DenseOfArray

FsDNN/Constants.fs

@@ -0,0 +1,6 @@
+namespace KS.FsDNN
+
+module Constants =
+
+  [<Literal>]
+  let DivideBy0Guard = 1E-12

FsDNN/Domain.fs

@@ -0,0 +1,24 @@
+namespace KS.FsDNN
+
+open MathNet.Numerics.LinearAlgebra
+
+[<AutoOpen>]
+module Domain =
+
+  type InputLayer =
+    { nx: int }
+
+  type Layer =
+    | FullyConnected of {| n: int |}
+
+  type LossLayer =
+    | SoftMax of {| nc: int |}
+
+  type Transformer =
+    { n: int
+      W: Matrix<double>
+      b: Matrix<double>
+      forward: Matrix<double> -> Matrix<double> }
+
+  type Net =
+    { Transformers: Transformer list }

FsDNN/FsDNN.fsproj

@@ -2,10 +2,15 @@
 
     <PropertyGroup>
         <TargetFramework>netstandard2.0</TargetFramework>
+        <AssemblyName>KS.FsDNN</AssemblyName>
+        <RootNamespace>KS.FsDNN</RootNamespace>
     </PropertyGroup>
 
     <ItemGroup>
-        <Compile Include="Library.fs"/>
+        <Compile Include="P2DFunTK.fs" />
+        <Compile Include="Constants.fs" />
+        <Compile Include="Domain.fs" />
+        <Compile Include="Net.fs" />
     </ItemGroup>
 
     <ItemGroup>

FsDNN/Net.fs

@@ -0,0 +1,33 @@
+namespace KS.FsDNN
+
+open System
+open MathNet.Numerics.Distributions
+open MathNet.Numerics.LinearAlgebra
+
+module Net =
+
+  let _makeHiddenTransformers seed (inputLayer: InputLayer) (hiddenLayers: Layer list) (lossLayer: LossLayer): Transformer list =
+    let n = match lossLayer with | SoftMax svm -> svm.nc
+
+    let W = DenseMatrix.random<double> n inputLayer.nx (Normal(0., 1., Random(seed + 1)))
+    let b = DenseMatrix.create n 1 0.
+
+    [ { n = n; W = W; b = b; forward = fun a -> W * a + b } ]
+
+  let _makeLossTransformer (lossLayer: LossLayer): Transformer =
+    let n = match lossLayer with | SoftMax svm -> svm.nc
+    let f (x: Matrix<double>) =
+      let expx = x.PointwiseExp();
+      expx.Divide(expx.ColumnSums().Sum() + Constants.DivideBy0Guard)
+    { n = n; W = null; b = null; forward = f }
+
+  let makeLayers seed (inputLayer: InputLayer) (hiddenLayers: Layer list) (lossLayer: LossLayer) =
+    let its = { n = inputLayer.nx; W = null; b = null; forward = id }
+    let hts = _makeHiddenTransformers seed inputLayer hiddenLayers lossLayer
+    let lts = _makeLossTransformer lossLayer
+
+    { Transformers = List.concat [ [ its ]; hts; [ lts ] ] }
+
+  let forward (net: Net) (X: Matrix<double>): Matrix<double> =
+    net.Transformers
+    |> List.fold (fun acc e -> e.forward acc) X