workable for 1 server and 1 worker

example/MNIST/MNIST.conf

@@ -4,7 +4,7 @@ data = train
 iter = mnist
     path_img = "./data/train-images-idx3-ubyte.gz"
     path_label = "./data/train-labels-idx1-ubyte.gz"
-    shuffle = 1
+    shuffle = 0
 iter = end
 # evaluation iterator
 eval = test
@@ -30,10 +30,10 @@ batch_size = 100
 
 ## global parameters
 dev = cpu
-save_model = 15
+save_model = 1
 max_round = 15
 num_round = 15
-train_eval = 1
+train_eval = 15
 random_type = gaussian
 ## learning parameters
 eta = 0.1

src/cxxnet_main.cpp

@@ -55,6 +55,7 @@ class CXXNetLearnTask {
       return 0;
     }
 
+    printf("%s\n", argv[1]);
     utils::ConfigIterator itr(argv[1]);
     while (itr.Next()) {
       this->SetParam(itr.name(), itr.val());
@@ -65,6 +66,7 @@ class CXXNetLearnTask {
         this->SetParam(name, val);
       }
     }
+    for (auto r : cfg) printf("%s\t%s\n", r.first.c_str(), r.second.c_str());
     this->Init();
     if (!silent) {
       printf("initializing end, start working\n");
@@ -107,6 +109,7 @@ class CXXNetLearnTask {
         this->LoadModel();
       }
     }
+    DD << "..";
     this->CreateIterators();
   }
   // load in latest model from model_folder

src/global.h

@@ -27,4 +27,8 @@ typedef mshadow::index_t index_t;
 typedef mshadow::default_real_t real_t;
 } // namespace cxxnet
 
+// debug
+#include "glog/logging.h"
+#define DD LOG(ERROR)
+
 #endif  // CXXNET_GLOBAL_H_

src/io/iter_mnist-inl.hpp

@@ -14,6 +14,7 @@ namespace cxxnet {
 class MNISTIterator: public IIterator<DataBatch> {
  public:
   MNISTIterator(void) {
+    DD  << "..";
     img_.dptr_ = NULL;
     mode_ = 1;
     inst_offset_ = 0;
@@ -25,15 +26,16 @@ class MNISTIterator: public IIterator<DataBatch> {
   }
   virtual void SetParam(const char *name, const char *val) {
     if (!strcmp(name, "silent")) silent_ = atoi(val);
-    if (!strcmp(name, "batch_size"))   batch_size_ = (index_t)atoi(val); 
+    if (!strcmp(name, "batch_size"))   batch_size_ = (index_t)atoi(val);
     if (!strcmp(name, "input_flat"))   mode_ = atoi(val);
     if (!strcmp(name, "shuffle")) shuffle_ = atoi(val);
     if (!strcmp(name, "index_offset")) inst_offset_ = atoi(val);
     if (!strcmp(name, "path_img"))     path_img = val;
-    if (!strcmp(name, "path_label"))   path_label = val;            
+    if (!strcmp(name, "path_label"))   path_label = val;
   }
   // intialize iterator loads data in
   virtual void Init(void) {
+    DD << "init";
     this->LoadImage();
     this->LoadLabel();
     if (mode_ == 1) {
@@ -47,7 +49,7 @@ class MNISTIterator: public IIterator<DataBatch> {
     if (shuffle_) this->Shuffle();
     if (silent_ == 0) {
       mshadow::Shape<4> s = out_.data.shape_;
-      printf("MNISTIterator: load %u images, shuffle=%d, shape=%u,%u,%u,%u\n", 
+      printf("MNISTIterator: load %u images, shuffle=%d, shape=%u,%u,%u,%u\n",
              (unsigned)img_.size(0), shuffle_, s[0], s[1], s[2], s[3]);
     }
   }
@@ -75,10 +77,10 @@ class MNISTIterator: public IIterator<DataBatch> {
     int image_count = ReadInt(gzimg);
     int image_rows  = ReadInt(gzimg);
     int image_cols  = ReadInt(gzimg);
-            
+
     img_.shape_ = mshadow::Shape3(image_count, image_rows, image_cols);
     img_.stride_ = img_.size(2);
-            
+
     // allocate continuous memory
     img_.dptr_ = new float[img_.MSize()];
     for (int i = 0; i < image_count; ++i) {
@@ -90,7 +92,7 @@ class MNISTIterator: public IIterator<DataBatch> {
     }
     // normalize to 0-1
     img_ *= 1.0f / 256.0f;
-  }        
+  }
   inline void LoadLabel(void) {
     utils::GzFile gzlabel(path_label.c_str(), "rb");
     ReadInt(gzlabel);
@@ -126,7 +128,7 @@ class MNISTIterator: public IIterator<DataBatch> {
   int silent_;
   // path
   std::string path_img, path_label;
-  // output 
+  // output
   DataBatch out_;
   // whether do shuffle
   int shuffle_;
@@ -136,14 +138,14 @@ class MNISTIterator: public IIterator<DataBatch> {
   index_t loc_;
   // batch size
   index_t batch_size_;
-  // image content 
+  // image content
   mshadow::Tensor<cpu,3> img_;
   // label content
   std::vector<float> labels_;
   // instance index offset
   unsigned inst_offset_;
   // instance index
-  std::vector<unsigned> inst_; 
+  std::vector<unsigned> inst_;
 }; //class MNISTIterator
 }  // namespace cxxnet
 #endif  // CXXNET_ITER_MNIST_INL_HPP_

src/nnet/neural_net-inl.hpp

@@ -127,7 +127,7 @@ struct NeuralNet {
                        bool need_update,
                        long update_epoch) {
     for (size_t i = connections.size(); i > 0; --i) {
-      layer::Connection<xpu> &c = connections[i - 1];      
+      layer::Connection<xpu> &c = connections[i - 1];
       for (size_t j = 0; j < updaters[i - 1].size(); ++j) {
         updaters[i - 1][j]->BeforeBackprop(c.nodes_in, c.nodes_out);
       }

src/nnet/nnet_config.h

@@ -14,12 +14,13 @@
 #include "../layer/layer.h"
 #include "../utils/utils.h"
 #include "../utils/io.h"
+// #include "glog/logging.h"
 
 namespace cxxnet {
 namespace nnet {
 /*!
  * \brief this is an object that records the configuration of a neural net
- *    it is used to store the network structure, and reads in configuration 
+ *    it is used to store the network structure, and reads in configuration
  *    that associates with each of the layers
  */
 struct NetConfig {
@@ -39,7 +40,7 @@ struct NetConfig {
     NetParam(void) {
       memset(reserved, 0, sizeof(reserved));
       num_nodes = 0;
-      num_layers = 0;      
+      num_layers = 0;
       input_shape = mshadow::Shape3(0, 0, 0);
       init_end = 0;
     }
@@ -48,16 +49,16 @@ struct NetConfig {
   struct LayerInfo {
     /*! \brief type of layer */
     layer::LayerType type;
-    /*! 
-     * \brief the index of primary layer, 
+    /*!
+     * \brief the index of primary layer,
      *  this field is only used when layer type is kSharedLayer
      */
     int primary_layer_index;
     /*! \brief layer name */
     std::string name;
     /*! \brief input node index */
     std::vector<int> nindex_in;
-    /*! \brief output node node index */    
+    /*! \brief output node node index */
     std::vector<int> nindex_out;
     LayerInfo(void) : primary_layer_index(-1), name() {
     }
@@ -139,7 +140,7 @@ struct NetConfig {
       utils::Check(fi.Read(&layers[i].name), "NetConfig: invalid model file");
       utils::Check(fi.Read(&layers[i].nindex_in), "NetConfig: invalid model file");
       utils::Check(fi.Read(&layers[i].nindex_out), "NetConfig: invalid model file");
-      if (layers[i].type == layer::kSharedLayer) { 
+      if (layers[i].type == layer::kSharedLayer) {
         utils::Check(layers[i].name.length() == 0, "SharedLayer must not have name");
       } else {
         utils::Check(layer_name_map.count(layers[i].name) == 0,
@@ -150,20 +151,23 @@ struct NetConfig {
     this->ClearConfig();
   }
   /*!
-   * \brief setup configuration, using the config string pass in 
+   * \brief setup configuration, using the config string pass in
    */
   inline void Configure(const std::vector< std::pair<std::string, std::string> > &cfg) {
+    // LOG(ERROR) << this;
+    // for (auto c : cfg)  LOG(ERROR) << c.first << "\t" << c.second;
     this->ClearConfig();
     // whether in net config mode
     int netcfg_mode = 0;
     // remembers what is the last top node
     int cfg_top_node = 0;
     // current configuration layer index
-    int cfg_layer_index = 0;   
+    int cfg_layer_index = 0;
     for (size_t i = 0; i < cfg.size(); ++i) {
       const char *name = cfg[i].first.c_str();
       const char *val = cfg[i].second.c_str();
-      if (param.init_end == 0) { 
+      // printf("%s \t %s \n", name, val);
+      if (param.init_end == 0) {
         if (!strcmp( name, "input_shape")) {
           unsigned x, y, z;
           utils::Check(sscanf(val, "%u,%u,%u", &z, &y, &x) == 3,
@@ -174,7 +178,7 @@ struct NetConfig {
       if (netcfg_mode != 2) {
         if (!strcmp(name, "updater")) updater_type = val;
         if (!strcmp(name, "sync")) sync_type = val;
-      }      
+      }
       if (!strcmp(name, "netconfig") && !strcmp(val, "start")) netcfg_mode = 1;
       if (!strcmp(name, "netconfig") && !strcmp(val, "end")) netcfg_mode = 0;
       if (!strncmp(name, "layer[", 6)) {
@@ -185,7 +189,7 @@ struct NetConfig {
           layers.push_back(info);
           layercfg.resize(layers.size());
         } else {
-          utils::Check(cfg_layer_index < static_cast<int>(layers.size()), "config layer index exceed bound");         
+          utils::Check(cfg_layer_index < static_cast<int>(layers.size()), "config layer index exceed bound");
           utils::Check(info == layers[cfg_layer_index],
                        "config setting does not match existing network structure");
         }
@@ -205,7 +209,7 @@ struct NetConfig {
     }
     if (param.init_end == 0) this->InitNet();
   }
-  
+
  private:
   // configuration parser to parse layer info, support one to to one connection for now
   // extend this later to support multiple connections
@@ -218,7 +222,7 @@ struct NetConfig {
     if (sscanf(name, "layer[+%d]", &b) == 1) {
       a = top_node; b += top_node;
       inf.nindex_in.push_back(a);
-      inf.nindex_out.push_back(b);    
+      inf.nindex_out.push_back(b);
     } else if (sscanf(name, "layer[%s", src) == 1) {
       char* dst = strchr(src, '-');
       dst += 2;
@@ -244,17 +248,20 @@ struct NetConfig {
     }
     if (inf.type == layer::kSharedLayer) {
       utils::Check(s_tag.length() != 0, "shared layer must specify tag of layer to share with");
-      utils::Check(layer_name_map.count(s_tag) != 0, 
+      utils::Check(layer_name_map.count(s_tag) != 0,
                    "shared layer tag %s is not defined before", s_tag.c_str());
       inf.primary_layer_index = layer_name_map[s_tag];
     } else {
       if (s_tag.length() != 0) {
         if (layer_name_map.count(s_tag) != 0) {
+          // LOG(ERROR) << layer_name_map[s_tag] << " " << cfg_layer_index << " " <<  s_tag.c_str();
+          CHECK_EQ(layer_name_map[s_tag],cfg_layer_index);
           utils::Check(layer_name_map[s_tag] == cfg_layer_index,
                        "layer name in the configuration file do not "\
                        "match the name stored in model");
         } else {
           layer_name_map[s_tag] = cfg_layer_index;
+          // LOG(ERROR) << layer_name_map[s_tag] << " " << cfg_layer_index << " " <<  s_tag.c_str();
         }
         inf.name = s_tag;
       }

src/nnet/nnet_impl-inl.hpp

@@ -60,17 +60,24 @@ class CXXNetThreadTrainer : public INetTrainer {
     cfg.push_back(std::make_pair(std::string(name), std::string(val)));
   }
   virtual void InitModel(void) {
+    DD << "xxx";
     this->InitNet();
+    DD << "xxx";
     nets_[0]->InitModel();
+    DD << "xxx";
     nets_[0]->WaitJob();
+    DD << "xxx";
     this->Save2ModelBlob();
+    DD << "xxx";
     for(size_t i = 1; i < nets_.size(); ++i) {
       utils::MemoryBufferStream fs(&model_blob_);
       nets_[i]->LoadModel(fs);
       nets_[i]->WaitJob();
     }
+    DD << "xxx";
     this->InitTemp();
 
+    DD << "xxx";
   }
   virtual void SaveModel(utils::IStream &fo) {
     this->Save2ModelBlob();
@@ -101,13 +108,13 @@ class CXXNetThreadTrainer : public INetTrainer {
     mshadow::Shape<4> oshape = out_temp.shape_;
     oshape[0] = data.batch_size;
     out_temp.Resize(oshape);
-    
+
     const size_t ndevice = devices_.size();
     mshadow::index_t step = std::max((batch_size + ndevice - 1) / ndevice, 1UL);
-    
+
     bool need_sync = sample_counter % update_period == 0;
     bool need_update = (sample_counter + 1) % update_period == 0;
-    
+
     for (mshadow::index_t i = nets_.size(); i != 0; --i) {
       mshadow::index_t begin = std::min((i - 1) * step, data.batch_size);
       mshadow::index_t end = std::min(i * step, data.batch_size);
@@ -227,7 +234,7 @@ class CXXNetThreadTrainer : public INetTrainer {
     utils::Assert(nets_.size() == 0, "net must be empty before this");
     net_cfg.Configure(cfg);
     if (devices_.size() == 0) devices_.push_back(0);
-    size_t ndevice = devices_.size(); 
+    size_t ndevice = devices_.size();
     mshadow::index_t step = std::max((batch_size + ndevice - 1) / ndevice, 1UL);
     while (step * (devices_.size() - 1) >= batch_size) {
       devices_.pop_back();

src/nnet/nnet_ps_server.cpp

@@ -8,6 +8,13 @@
 #include "../layer/param.h"
 #include "../utils/config.h"
 #include "../updater/updater.h"
+#include "glog/logging.h"
+#include "gflags/gflags.h"
+#include "ps.h"
+
+namespace PS {
+DECLARE_string(app_file);
+} // namespace PS
 
 namespace cxxnet {
 namespace nnet {
@@ -26,11 +33,18 @@ class NetServer : public mshadow::ps::ICustomServer<real_t> {
     cfgvec.push_back(std::make_pair(std::string(name), std::string(val)));
   }
   virtual void Init(int rank, const std::string &conf) {
-    std::stringstream ss(conf);
-    utils::ConfigStreamReader reader(ss);
-    while (reader.Next()) {
-      this->SetParam(reader.name(), reader.val());
+    // FIXME
+    // std::stringstream ss(conf);
+    // utils::ConfigStreamReader reader(ss);
+
+    // if (PS::Postoffice::instance().app()->isServer()) {
+    if (PS::FLAGS_app_file.size()) {
+      utils::ConfigIterator reader(PS::FLAGS_app_file.c_str());
+      while (reader.Next()) {
+        this->SetParam(reader.name(), reader.val());
+      }
     }
+
     // start configure settings
     cfg.Configure(cfgvec);
     rnd.Seed(seed + rank * 17);
@@ -47,6 +61,7 @@ class NetServer : public mshadow::ps::ICustomServer<real_t> {
          updater::DecodeTag(key));
     e.is_bias = !strcmp(updater::DecodeTag(key), "bias");
     const int i = key / updater::kDataKeyStep;
+CHECK_LT(i, cfg.param.num_layers) <<  "layer index exceed bound";
     utils::Assert(i < cfg.param.num_layers, "layer index exceed bound");
     e.layer_type = cfg.layers[i].type;
     for (size_t j = 0; j < cfg.defcfg.size(); ++j) {
@@ -115,7 +130,7 @@ class NetServer : public mshadow::ps::ICustomServer<real_t> {
       epoch += 1;
     }
   };
-  
+
  private:
   int seed;
   mshadow::Random<cpu> rnd;