Neural networks

.classpath

@@ -31,5 +31,9 @@
 	<classpathentry kind="lib" path="lib/hadoop/cdh3/hadoop-core-0.20.2-cdh3u6.jar" sourcepath="lib/hadoop/cdh3/hadoop-core-0.20.2-cdh3u6-sources.jar"/>
 	<classpathentry kind="lib" path="lib/log4j/log4j-1.2.15.jar"/>
 	<classpathentry kind="con" path="org.eclipse.jdt.USER_LIBRARY/Python 2.7.2 (/System/Library/Frameworks/Python.framework/Versions/2.7/bin/python) interpreter library"/>
+	<classpathentry kind="lib" path="lib/jogamp/jocl-natives-linux-amd64.jar"/>
+	<classpathentry kind="lib" path="lib/jogamp/jocl.jar" sourcepath="lib/jogamp/jocl-sources.jar"/>
+	<classpathentry kind="lib" path="lib/jogamp/gluegen-rt-natives-linux-amd64.jar"/>
+	<classpathentry kind="lib" path="lib/jogamp/gluegen-rt.jar" sourcepath="lib/jogamp/gluegen-rt-sources.jar"/>
 	<classpathentry kind="output" path="target/classes"/>
 </classpath>

.gitignore

@@ -23,7 +23,6 @@ sandbox
 syn_datasets
 target/
 .settings/*
-cypof*
 /libpeerconnection.log
 src/main/java/water/BuildVersion.java
 hadoop/build.log

build.sh

@@ -36,7 +36,7 @@ JAR_ROOT=lib
 
 # additional dependencies, relative to this file, but all dependencies should be
 # inside the JAR_ROOT tree so that they are packed to the jar file properly
-DEPENDENCIES="${JAR_ROOT}/jama/*${SEP}${JAR_ROOT}/apache/*${SEP}${JAR_ROOT}/junit/*${SEP}${JAR_ROOT}/gson/*${SEP}${JAR_ROOT}/javassist.jar${SEP}${JAR_ROOT}/poi/*${SEP}${JAR_ROOT}/s3/*${SEP}${JAR_ROOT}/jets3t/*${SEP}${JAR_ROOT}/log4j/*${SEP}${JAR_ROOT}/mockito/*"
+DEPENDENCIES="${JAR_ROOT}/jama/*${SEP}${JAR_ROOT}/apache/*${SEP}${JAR_ROOT}/junit/*${SEP}${JAR_ROOT}/gson/*${SEP}${JAR_ROOT}/javassist.jar${SEP}${JAR_ROOT}/poi/*${SEP}${JAR_ROOT}/s3/*${SEP}${JAR_ROOT}/jets3t/*${SEP}${JAR_ROOT}/log4j/*${SEP}${JAR_ROOT}/mockito/*${SEP}${JAR_ROOT}/jogamp/*"
 
 DEFAULT_HADOOP_VERSION="cdh3"
 OUTDIR="target"

experiments/.classpath

@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<classpath>
+	<classpathentry kind="src" path="src/main/java"/>
+	<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
+	<classpathentry combineaccessrules="false" kind="src" path="/h2o"/>
+	<classpathentry kind="output" path="target/classes"/>
+</classpath>

experiments/.project

@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<projectDescription>
+	<name>experiments</name>
+	<comment></comment>
+	<projects>
+	</projects>
+	<buildSpec>
+		<buildCommand>
+			<name>org.eclipse.jdt.core.javabuilder</name>
+			<arguments>
+			</arguments>
+		</buildCommand>
+	</buildSpec>
+	<natures>
+		<nature>org.eclipse.jdt.core.javanature</nature>
+	</natures>
+</projectDescription>

experiments/.settings/org.eclipse.jdt.ui.prefs

@@ -0,0 +1,62 @@
+eclipse.preferences.version=1
+editor_save_participant_org.eclipse.jdt.ui.postsavelistener.cleanup=true
+formatter_profile=_0xdata
+formatter_settings_version=12
+org.eclipse.jdt.ui.ignorelowercasenames=true
+org.eclipse.jdt.ui.importorder=java;javax;org;com;
+org.eclipse.jdt.ui.javadoc=false
+org.eclipse.jdt.ui.ondemandthreshold=3
+org.eclipse.jdt.ui.staticondemandthreshold=3
+org.eclipse.jdt.ui.text.custom_code_templates=<?xml version\="1.0" encoding\="UTF-8" standalone\="no"?><templates><template autoinsert\="true" context\="gettercomment_context" deleted\="false" description\="Comment for getter method" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.gettercomment" name\="gettercomment">/**\n * @return the ${bare_field_name}\n */</template><template autoinsert\="true" context\="settercomment_context" deleted\="false" description\="Comment for setter method" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.settercomment" name\="settercomment">/**\n * @param ${param} the ${bare_field_name} to set\n */</template><template autoinsert\="true" context\="constructorcomment_context" deleted\="false" description\="Comment for created constructors" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.constructorcomment" name\="constructorcomment">/**\n * ${tags}\n */</template><template autoinsert\="true" context\="filecomment_context" deleted\="false" description\="Comment for created Java files" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.filecomment" name\="filecomment">/**\n * \n */</template><template autoinsert\="true" context\="typecomment_context" deleted\="false" description\="Comment for created types" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.typecomment" name\="typecomment">/**\n * @author ${user}\n *\n * ${tags}\n */</template><template autoinsert\="true" context\="fieldcomment_context" deleted\="false" description\="Comment for fields" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.fieldcomment" name\="fieldcomment">/**\n * \n */</template><template autoinsert\="true" context\="methodcomment_context" deleted\="false" description\="Comment for non-overriding methods" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.methodcomment" name\="methodcomment">/**\n * ${tags}\n */</template><template autoinsert\="true" context\="overridecomment_context" deleted\="false" description\="Comment for overriding methods" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.overridecomment" name\="overridecomment">/* (non-Javadoc)\n * ${see_to_overridden}\n */</template><template autoinsert\="true" context\="delegatecomment_context" deleted\="false" description\="Comment for delegate methods" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.delegatecomment" name\="delegatecomment">/**\n * ${tags}\n * ${see_to_target}\n */</template><template autoinsert\="true" context\="newtype_context" deleted\="false" description\="Newly created files" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.newtype" name\="newtype">${filecomment}\n${package_declaration}\n\n${typecomment}\n${type_declaration}</template><template autoinsert\="true" context\="classbody_context" deleted\="false" description\="Code in new class type bodies" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.classbody" name\="classbody">\n</template><template autoinsert\="true" context\="interfacebody_context" deleted\="false" description\="Code in new interface type bodies" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.interfacebody" name\="interfacebody">\n</template><template autoinsert\="true" context\="enumbody_context" deleted\="false" description\="Code in new enum type bodies" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.enumbody" name\="enumbody">\n</template><template autoinsert\="true" context\="annotationbody_context" deleted\="false" description\="Code in new annotation type bodies" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.annotationbody" name\="annotationbody">\n</template><template autoinsert\="false" context\="catchblock_context" deleted\="false" description\="Code in new catch blocks" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.catchblock" name\="catchblock">// ${todo} Auto-generated catch block\nthrow new RuntimeException(${exception_var});\n</template><template autoinsert\="false" context\="methodbody_context" deleted\="false" description\="Code in created method stubs" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.methodbody" name\="methodbody">throw new RuntimeException("${todo} Auto-generated method stub");</template><template autoinsert\="true" context\="constructorbody_context" deleted\="false" description\="Code in created constructor stubs" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.constructorbody" name\="constructorbody">${body_statement}\n// ${todo} Auto-generated constructor stub</template><template autoinsert\="true" context\="getterbody_context" deleted\="false" description\="Code in created getters" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.getterbody" name\="getterbody">return ${field};</template><template autoinsert\="true" context\="setterbody_context" deleted\="false" description\="Code in created setters" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.setterbody" name\="setterbody">${field} \= ${param};</template></templates>
+sp_cleanup.add_default_serial_version_id=true
+sp_cleanup.add_generated_serial_version_id=false
+sp_cleanup.add_missing_annotations=false
+sp_cleanup.add_missing_deprecated_annotations=true
+sp_cleanup.add_missing_methods=false
+sp_cleanup.add_missing_nls_tags=false
+sp_cleanup.add_missing_override_annotations=true
+sp_cleanup.add_missing_override_annotations_interface_methods=true
+sp_cleanup.add_serial_version_id=false
+sp_cleanup.always_use_blocks=true
+sp_cleanup.always_use_parentheses_in_expressions=false
+sp_cleanup.always_use_this_for_non_static_field_access=false
+sp_cleanup.always_use_this_for_non_static_method_access=false
+sp_cleanup.convert_to_enhanced_for_loop=false
+sp_cleanup.correct_indentation=false
+sp_cleanup.format_source_code=false
+sp_cleanup.format_source_code_changes_only=false
+sp_cleanup.make_local_variable_final=false
+sp_cleanup.make_parameters_final=false
+sp_cleanup.make_private_fields_final=true
+sp_cleanup.make_type_abstract_if_missing_method=false
+sp_cleanup.make_variable_declarations_final=false
+sp_cleanup.never_use_blocks=false
+sp_cleanup.never_use_parentheses_in_expressions=true
+sp_cleanup.on_save_use_additional_actions=true
+sp_cleanup.organize_imports=false
+sp_cleanup.qualify_static_field_accesses_with_declaring_class=false
+sp_cleanup.qualify_static_member_accesses_through_instances_with_declaring_class=true
+sp_cleanup.qualify_static_member_accesses_through_subtypes_with_declaring_class=true
+sp_cleanup.qualify_static_member_accesses_with_declaring_class=false
+sp_cleanup.qualify_static_method_accesses_with_declaring_class=false
+sp_cleanup.remove_private_constructors=true
+sp_cleanup.remove_trailing_whitespaces=true
+sp_cleanup.remove_trailing_whitespaces_all=true
+sp_cleanup.remove_trailing_whitespaces_ignore_empty=false
+sp_cleanup.remove_unnecessary_casts=false
+sp_cleanup.remove_unnecessary_nls_tags=false
+sp_cleanup.remove_unused_imports=false
+sp_cleanup.remove_unused_local_variables=false
+sp_cleanup.remove_unused_private_fields=true
+sp_cleanup.remove_unused_private_members=false
+sp_cleanup.remove_unused_private_methods=true
+sp_cleanup.remove_unused_private_types=true
+sp_cleanup.sort_members=false
+sp_cleanup.sort_members_all=false
+sp_cleanup.use_blocks=false
+sp_cleanup.use_blocks_only_for_return_and_throw=false
+sp_cleanup.use_parentheses_in_expressions=false
+sp_cleanup.use_this_for_non_static_field_access=false
+sp_cleanup.use_this_for_non_static_field_access_only_if_necessary=true
+sp_cleanup.use_this_for_non_static_method_access=false
+sp_cleanup.use_this_for_non_static_method_access_only_if_necessary=true

experiments/src/main/java/hex/Histogram.java

@@ -0,0 +1,169 @@
+package hex;
+
+import java.util.ArrayList;
+import java.util.concurrent.*;
+
+import javafx.application.Platform;
+import javafx.beans.value.ChangeListener;
+import javafx.beans.value.ObservableValue;
+import javafx.collections.FXCollections;
+import javafx.collections.ObservableList;
+import javafx.embed.swing.JFXPanel;
+import javafx.event.ActionEvent;
+import javafx.event.EventHandler;
+import javafx.scene.Scene;
+import javafx.scene.chart.LineChart;
+import javafx.scene.chart.NumberAxis;
+import javafx.scene.control.*;
+import javafx.scene.layout.*;
+import javafx.stage.Stage;
+
+import javax.swing.SwingUtilities;
+
+public class Histogram extends LineChart {
+  private static final int SLICES = 64;
+
+  private static final ArrayList<Histogram> _instances = new ArrayList<Histogram>();
+  private static final ScheduledExecutorService _executor = Executors.newSingleThreadScheduledExecutor();
+  private static CheckBox _auto;
+
+  private final float[] _data;
+  private final ObservableList<Data<Float, Float>> _list = FXCollections.observableArrayList();
+
+  static void init() {
+    final CountDownLatch latch = new CountDownLatch(1);
+    SwingUtilities.invokeLater(new Runnable() {
+      public void run() {
+        initFromSwingThread();
+        latch.countDown();
+      }
+    });
+    try {
+      latch.await();
+    } catch( InterruptedException e ) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  static void initFromSwingThread() {
+    new JFXPanel(); // initializes JavaFX environment
+  }
+
+  public static void build(final Layer[] ls) {
+    Platform.runLater(new Runnable() {
+      @Override public void run() {
+        VBox v = new VBox();
+        for( int i = ls.length - 1; i > 0; i-- ) {
+          HBox h = new HBox();
+          h.getChildren().add(new Histogram("Layer " + i + " A", ls[i]._a));
+          h.getChildren().add(new Histogram("E", ls[i]._e));
+          v.getChildren().add(h);
+
+          h = new HBox();
+          h.getChildren().add(new Histogram("Layer " + i + " W", ls[i]._w));
+          h.getChildren().add(new Histogram("B", ls[i]._b));
+          v.getChildren().add(h);
+
+          h = new HBox();
+          h.getChildren().add(new Histogram("Layer " + i + " W S", ls[i]._wSpeed));
+          h.getChildren().add(new Histogram("W B", ls[i]._bSpeed));
+          v.getChildren().add(h);
+
+          if( ls[i]._v != null ) {
+            h = new HBox();
+            h.getChildren().add(new Histogram("Layer " + i + " V", ls[i]._v));
+            h.getChildren().add(new Histogram("Gradient " + i + " V", ls[i]._gv));
+            v.getChildren().add(h);
+          }
+        }
+        Stage stage = new Stage();
+        BorderPane root = new BorderPane();
+        ToolBar toolbar = new ToolBar();
+
+        Button refresh = new Button("Refresh");
+        refresh.setOnAction(new EventHandler<ActionEvent>() {
+          @Override public void handle(ActionEvent e) {
+            refresh();
+          }
+        });
+        toolbar.getItems().add(refresh);
+
+        _auto = new CheckBox("Auto");
+        _auto.selectedProperty().addListener(new ChangeListener<Boolean>() {
+          public void changed(ObservableValue<? extends Boolean> ov, Boolean old_val, Boolean new_val) {
+            refresh();
+          }
+        });
+        toolbar.getItems().add(_auto);
+
+        root.setTop(toolbar);
+        ScrollPane scroll = new ScrollPane();
+        scroll.setContent(v);
+        root.setCenter(scroll);
+        Scene scene = new Scene(root);
+        stage.setScene(scene);
+        stage.setWidth(1500);
+        stage.setHeight(1100);
+        stage.show();
+
+        scene.getWindow().onCloseRequestProperty().addListener(new ChangeListener() {
+          @Override public void changed(ObservableValue arg0, Object arg1, Object arg2) {
+            _auto.selectedProperty().set(false);
+          }
+        });
+        refresh();
+      }
+    });
+  }
+
+  public Histogram(String title, float[] data) {
+    super(new NumberAxis(), new NumberAxis());
+    _data = data;
+
+    ObservableList<Series<Float, Float>> series = FXCollections.observableArrayList();
+    for( int i = 0; i < SLICES; i++ )
+      _list.add(new Data<Float, Float>(0f, 0f));
+    series.add(new LineChart.Series<Float, Float>(title, _list));
+    setData(series);
+    setPrefWidth(600);
+    setPrefHeight(250);
+
+    _instances.add(this);
+  }
+
+  static void refresh() {
+    for( Histogram h : _instances ) {
+      if( h._data != null ) {
+        float[] data = h._data.clone();
+        float min = Float.MAX_VALUE, max = Float.MIN_VALUE;
+        for( int i = 0; i < data.length; i++ ) {
+          max = Math.max(max, data[i]);
+          min = Math.min(min, data[i]);
+        }
+        int[] counts = new int[SLICES];
+        float inc = (max - min) / (SLICES - 1);
+        for( int i = 0; i < data.length; i++ )
+          counts[(int) Math.floor((data[i] - min) / inc)]++;
+
+        for( int i = 0; i < SLICES; i++ ) {
+          Data<Float, Float> point = h._list.get(i);
+          point.setXValue(min + inc * i);
+          point.setYValue((float) counts[i] / data.length);
+        }
+      }
+    }
+
+    if( _auto.selectedProperty().get() ) {
+      _executor.schedule(new Runnable() {
+
+        @Override public void run() {
+          Platform.runLater(new Runnable() {
+            @Override public void run() {
+              refresh();
+            }
+          });
+        }
+      }, 1000, TimeUnit.MILLISECONDS);
+    }
+  }
+}

experiments/src/main/java/hex/IrisSearch.java

@@ -0,0 +1,54 @@
+package hex;
+
+import hex.Layer.Input;
+import water.Sandbox;
+
+public class IrisSearch extends NeuralNetIrisTest {
+  public static void main(String[] args) throws Exception {
+    water.Boot.main(UserMain.class, args);
+  }
+
+  public static class UserMain {
+    public static void main(String[] args) throws Exception {
+      Sandbox.localCloud(1, true, args);
+
+      IrisSearch test = new IrisSearch();
+      test.run();
+    }
+  }
+
+  @Override public void run() throws Exception {
+    load();
+
+    ParamsSearch search = new ParamsSearch();
+    double best = Double.MAX_VALUE;
+    for( ;; ) {
+      float rate = 0.01f;
+      float momentum = .1f;
+      int epochs = 4;
+      create(rate, momentum);
+
+      Trainer trainer = new Trainer.Direct(_ls);
+      trainer._batches = epochs * (int) _train._frame.numRows();
+      trainer._batch = 1;
+      search.run(_ls[1], _ls[2]);
+      trainer.run();
+      double error = eval(_ls, _train, trainer);
+      String m = "Error: " + error * 100 + " (Best: " + best + ")";
+      if( error < best ) {
+        best = error;
+        search.save();
+        System.out.println(m + ", Saved");
+      } else
+        System.out.println(m + ", Discarded");
+    }
+  }
+
+  static double eval(Layer[] ls, Input input, Trainer trainer) {
+    int correct = 0;
+    for( int n = 0; n < input._count; n++ )
+      if( test(ls, n) )
+        correct++;
+    return (input._count - (double) correct) / input._count;
+  }
+}