Merge commit 'f86c1b960eb800fffdcddf55e2b25a1c97da09ec' into develop

src/framework/featurespace.h

@@ -3,12 +3,15 @@
 
 #include <iostream>
 #include <vector>
-#include <cstdint>
+#include "boost/cstdint.hpp"
 #include "utils/smartmap.hpp"
 
 namespace ltp {
 namespace framework {
 
+using boost::uint32_t;
+using boost::int32_t;
+
 class FeatureSpaceIterator {
 public:
   FeatureSpaceIterator()

src/framework/parameter.h

@@ -3,14 +3,16 @@
 
 #include <iostream>
 #include <cstring>
-#include <cstdint>
+#include "boost/cstdint.hpp"
 #include "utils/math/sparsevec.h"
 #include "utils/math/featurevec.h"
 #include "utils/logging.hpp"
 
 namespace ltp {
 namespace framework {
 
+using boost::uint32_t;
+
 class Parameters {
 public:
   bool _enable_wrapper;

src/framework/serializable.h

@@ -2,11 +2,13 @@
 #define __LTP_FRAMEWORK_SERIALIZABLE_H__
 
 #include <iostream>
-#include <cstdint>
+#include "boost/cstdint.hpp"
 
 namespace ltp {
 namespace framework {
 
+using boost::uint32_t;
+
 class Serializable {
 protected:
   void write_uint(std::ostream & out, uint32_t val) const {

src/ner/CMakeLists.txt

@@ -2,28 +2,43 @@ include_directories (${SOURCE_DIR}/ ${THIRDPARTY_DIR}/boost/include/)
 
 set (ner_VERSION "0.0.2")
 set (ner_SRC
+    settings.h
+    decoder.h
     decoder.cpp
+    extractor.h
     extractor.cpp
+    ner.h
     ner.cpp)
 
 # -----------------------------------------------
 # STATIC LIBRARY
 # -----------------------------------------------
-add_library (ner_static_lib STATIC ner_dll.cpp ${ner_SRC})
+add_library (ner_static_lib STATIC
+    ner_dll.h
+    ner_dll.cpp
+    ${ner_SRC})
 set_target_properties (ner_static_lib PROPERTIES OUTPUT_NAME ner)
 
 # -----------------------------------------------
 # SHARED LIBRARY
 # -----------------------------------------------
-add_library (ner_shared_lib SHARED ner_dll.cpp ${ner_SRC})
+add_library (ner_shared_lib SHARED
+    ner_dll.h
+    ner_dll.cpp
+    ${ner_SRC})
 set_target_properties (ner_shared_lib PROPERTIES
     VERSION     ${ner_VERSION}
     OUTPUT_NAME ner)
 
 # ------------------------------------------------
 # EXECUTABLE
 # ------------------------------------------------
-add_executable (otner otner.cpp ner_frontend.cpp io.cpp ${ner_SRC})
+add_executable (otner otner.cpp
+    ner_frontend.h    
+    ner_frontend.cpp
+    io.h
+    io.cpp
+    ${ner_SRC})
 target_link_libraries (otner boost_program_options_static_lib)
 set_target_properties (otner PROPERTIES
     OUTPUT_NAME otner

src/parser.n/CMakeLists.txt

@@ -5,27 +5,45 @@ include_directories (${SOURCE_DIR}/
 
 set (nndepparser_VERSION "0.0.1")
 set (nndepparser_SRC
+    options.h
     options.cpp
+    instance.h
     instance.cpp
+    system.h
     system.cpp
+    classifier.h
     classifier.cpp
+    parser.h
     parser.cpp)
 
-add_library(parser_static_lib STATIC parser_dll.cpp ${nndepparser_SRC})
+add_library(parser_static_lib STATIC
+    parser_dll.h
+    parser_dll.cpp
+    ${nndepparser_SRC})
 set_target_properties (parser_static_lib    PROPERTIES
     OUTPUT_NAME     parser)
 
-add_library(parser_shared_lib SHARED parser_dll.cpp ${nndepparser_SRC})
+add_library(parser_shared_lib SHARED 
+    parser_dll.h
+    parser_dll.cpp
+    ${nndepparser_SRC})
 set_target_properties (parser_shared_lib    PROPERTIES
     VERSION         ${nndepparser_VERSION}
     OUTPUT_NAME     parser)
 
 # redirect output binary to tools/train
-add_executable (nndepparser main.cpp parser_frontend.cpp io.cpp ${nndepparser_SRC})
+add_executable (nndepparser 
+    main.cpp
+    parser_frontend.h
+    parser_frontend.cpp
+    io.h
+    io.cpp
+    ${nndepparser_SRC})
 target_link_libraries (nndepparser boost_program_options_static_lib)
 set_target_properties (nndepparser PROPERTIES
     OUTPUT_NAME nndepparser
     RUNTIME_OUTPUT_DIRECTORY ${TOOLS_DIR}/train/)
+
 configure_file (
     parser_dll.h
     ${INCLUDE_OUTPUT_PATH}/ltp/parser_dll.h)

src/parser.n/classifier.cpp

@@ -279,8 +279,8 @@ void NeuralNetworkClassifier::compute_gradient(
 
     /*arma::uvec classes_mask = arma::find(Y >= 0);*/
     Eigen::VectorXd __ = (Y.array() >= 0).select(
-        Eigen::VectorXd::Ones(hidden_layer_size),
-        Eigen::VectorXd::Zero(hidden_layer_size));
+        Eigen::VectorXd::Ones(nr_classes),
+        Eigen::VectorXd::Zero(nr_classes));
     double best = output(opt_class);
     output = __.asDiagonal() * Eigen::VectorXd((output.array() - best).exp());
     double sum1 = output(correct_class);

src/parser.n/parser_frontend.cpp

@@ -506,7 +506,9 @@ void NeuralNetworkParserFrontend::test(void) {
   size_t corr_heads = 0, corr_deprels = 0, nr_tokens = 0;
 
   timer t;
-  while (inst = reader.next()) {
+  while (true) {
+    inst = reader.next();
+    if (inst == NULL) { break; }
     predict((*inst), heads, deprels);
     writer.write(*inst, heads, deprels);
 

src/postagger/CMakeLists.txt

@@ -2,29 +2,44 @@ include_directories (${SOURCE_DIR}/ ${THIRDPARTY_DIR}/boost/include/)
 
 set (postagger_VERSION "0.0.3")
 set (postagger_SRC
+    settings.h
+    decoder.h
     decoder.cpp
+    extractor.h
     extractor.cpp
+    postagger.h
     postagger.cpp)
 
 # -----------------------------------------------
 # STATIC LIBRARY
 # -----------------------------------------------
-add_library (postagger_static_lib postag_dll.cpp ${postagger_SRC})
+add_library (postagger_static_lib
+    postag_dll.h
+    postag_dll.cpp
+    ${postagger_SRC})
 set_target_properties (postagger_static_lib PROPERTIES
     OUTPUT_NAME postagger)
 
 # -----------------------------------------------
 # SHARED LIBRARY
 # -----------------------------------------------
-add_library (postagger_shared_lib SHARED postag_dll.cpp ${postagger_SRC})
+add_library (postagger_shared_lib SHARED
+    postag_dll.h
+    postag_dll.cpp
+    ${postagger_SRC})
 set_target_properties (postagger_shared_lib PROPERTIES
     VERSION     ${postagger_VERSION}
     OUTPUT_NAME postagger)
 
 # ------------------------------------------------
 # EXECUTABLE
 # ------------------------------------------------
-add_executable (otpos otpos.cpp postagger_frontend.cpp io.cpp ${postagger_SRC})
+add_executable (otpos otpos.cpp
+    postagger_frontend.h
+    postagger_frontend.cpp
+    io.h
+    io.cpp
+    ${postagger_SRC})
 target_link_libraries (otpos boost_program_options_static_lib)
 set_target_properties (otpos PROPERTIES
     OUTPUT_NAME otpos

src/segmentor/CMakeLists.txt

@@ -3,16 +3,28 @@ include_directories ( ${SOURCE_DIR} ${THIRDPARTY_DIR}/boost/include )
 set (segment_VERSION "0.2.0")
 
 set (segment_SRC
+    settings.h
+    special_tokens.h
+    options.h
+    decoder.h
     decoder.cpp
+    preprocessor.h
     preprocessor.cpp
+    model.h
     model.cpp
+    extractor.h
     extractor.cpp
-    segmentor.cpp)
+    segmentor.h
+    segmentor.cpp
+    )
 
 # -----------------------------------------------
 # STATIC LIBRARY
 # -----------------------------------------------
-add_library (segmentor_static_lib segment_dll.cpp ${segment_SRC})
+add_library (segmentor_static_lib
+    segment_dll.h
+    segment_dll.cpp
+    ${segment_SRC})
 target_link_libraries (segmentor_static_lib boost_regex_static_lib)
 set_target_properties (segmentor_static_lib
     PROPERTIES
@@ -21,7 +33,10 @@ set_target_properties (segmentor_static_lib
 # -----------------------------------------------
 # SHARED LIBRARY
 # -----------------------------------------------
-add_library (segmentor_shared_lib SHARED segment_dll.cpp ${segment_SRC})
+add_library (segmentor_shared_lib SHARED
+    segment_dll.h
+    segment_dll.cpp
+    ${segment_SRC})
 target_link_libraries (segmentor_shared_lib boost_regex_shared_lib)
 set_target_properties (segmentor_shared_lib PROPERTIES
     VERSION     ${segment_VERSION}
@@ -32,12 +47,23 @@ link_directories ( ${LIBRARY_OUTPUT_PATH} )
 # -----------------------------------------------
 # TOOLKIT #1
 # -----------------------------------------------
-add_executable (otcws otcws.cpp segmentor_frontend.cpp partial_segmentation.cpp
-    customized_segmentor_frontend.cpp io.cpp ${segment_SRC})
+add_executable (otcws otcws.cpp
+    segmentor_frontend.h
+    segmentor_frontend.cpp
+    partial_segmentation.h
+    partial_segmentation.cpp
+    customized_segmentor_frontend.h
+    customized_segmentor_frontend.cpp
+    io.h
+    io.cpp
+    ${segment_SRC})
 target_link_libraries (otcws boost_regex_static_lib boost_program_options_static_lib)
 set_target_properties (otcws PROPERTIES  OUTPUT_NAME otcws
     RUNTIME_OUTPUT_DIRECTORY ${TOOLS_DIR}/train/)
 
+# -----------------------------------------------
+# CONFIG header FILE
+# -----------------------------------------------
 configure_file (
     segment_dll.h
     ${INCLUDE_OUTPUT_PATH}/ltp/segment_dll.h)

src/splitsnt/CMakeLists.txt

@@ -2,7 +2,7 @@ include_directories (${SOURCE_DIR})
 
 set (splitsnt_VERSION "0.0.1")
 
-add_library (splitsnt_static_lib SplitSentence.cpp)
+add_library (splitsnt_static_lib SplitSentence.h SplitSentence.cpp)
 
 set_target_properties (splitsnt_static_lib  PROPERTIES
     OUTPUT_NAME     splitsnt)

src/utils/smartmap.hpp

@@ -5,12 +5,15 @@
 #include <algorithm>
 #include <vector>
 #include <cstring>
-#include <cstdint>
+#include "boost/cstdint.hpp"
 #include "utils/hasher.hpp"
 
 namespace ltp {
 namespace utility {
 
+using boost::uint32_t;
+using boost::int32_t;
+
 struct __SmartMap_Hash_Node {
 public:
   unsigned int    __key_off;
@@ -645,7 +648,7 @@ class IndexableSmartMap : public SmartMap<int32_t> {
    *  @return   const char *  pointer to the key
    */
   const char* at(const size_t& i) const {
-    if (i >= 0 && i < _num_entries) {
+    if (i < _num_entries) {
       return SmartMap<int32_t>::_key_buffer + entries[i];
     } else {
       return 0;

src/utils/strutils.hpp

@@ -16,7 +16,7 @@ inline void trim(std::string& str) {
   size_t len = str.size();
   if (len == 0) { return; }
 
-  while (len -1 >=0 && (str[len-1] == ' '
+  while (len >= 1 && (str[len-1] == ' '
         || str[len-1]=='\t'
         || str[len-1] == '\r'
         || str[len-1] == '\n')) {