Merge pull request HIT-SCIR#207 from endyul/feature/json_support

json support in `ltp_server` and `ltp_test` [HIT-SCIR#195]

.gitignore

@@ -11,7 +11,7 @@
 # build       #
 ###############
 build
-config.h
+#config.h
 
 ###############
 # config      #

.travis.yml

@@ -1,13 +1,61 @@
-language:
-    - cpp
+sudo: required
+dist: precise
+language: cpp
 
-compiler:
-    - g++
+matrix:
+  include:
+    - compiler: gcc
+      addons:
+        apt:
+          sources:
+            - ubuntu-toolchain-r-test
+            - george-edison55-precise-backports
+          packages:
+            - g++-4.9
+            - cmake
+            - cmake-data
+      env: COMPILER=g++-4.9
+    - compiler: gcc
+      addons:
+        apt:
+          sources:
+            - ubuntu-toolchain-r-test
+            - george-edison55-precise-backports
+          packages:
+            - g++-5
+            - cmake
+            - cmake-data
+      env: COMPILER=g++-5
+    - compiler: clang
+      addons:
+        apt:
+          sources:
+            - ubuntu-toolchain-r-test
+            - llvm-toolchain-precise-3.6
+            - george-edison55-precise-backports
+          packages:
+            - clang-3.6
+            - cmake
+            - cmake-data
+      env: COMPILER=clang++-3.6
+    - compiler: clang
+      addons:
+        apt:
+          sources:
+            - ubuntu-toolchain-r-test
+            - llvm-toolchain-precise-3.7
+            - george-edison55-precise-backports
+          packages:
+            - clang-3.7
+            - cmake
+            - cmake-data
+      env: COMPILER=clang++-3.7
 
 before_install:
-    - sudo apt-get install cmake
+  - sudo apt-get update -qq
 
 script:
-    - ./configure
+    - mkdir -p build
+    - cd build
+    - cmake -DCMAKE_CXX_COMPILER=$COMPILER ..
     - make
-

CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required (VERSION 2.8.0)
+cmake_minimum_required (VERSION 2.8.8)
 project ("LTP - Language Technology Platform")
 
 # project attributes section

src/console/CMakeLists.txt

@@ -3,7 +3,8 @@ include_directories (./
     ${THIRDPARTY_DIR}/boost/include/
     ${THIRDPARTY_DIR}/tinythreadpp
     ${THIRDPARTY_DIR}/maxent
-    ${THIRDPARTY_DIR}/tinyxml)
+    ${THIRDPARTY_DIR}/tinyxml
+    ${THIRDPARTY_DIR}/jsoncpp/include)
 
 set (ltp_test_SRC ltp_test.cpp ${THIRDPARTY_DIR}/tinythreadpp/tinythread.cpp)
 
@@ -20,7 +21,8 @@ target_link_libraries (ltp_test
     srl_static_lib
     xml4nlp
     boost_regex_static_lib
-    boost_program_options_static_lib)
+    boost_program_options_static_lib
+    jsoncpp)
 
 add_executable (cws_cmdline cws_cmdline.cpp
     ${THIRDPARTY_DIR}/tinythreadpp/tinythread.cpp)

src/console/ltp_test.cpp

@@ -7,6 +7,7 @@
 #include "ltp/Ltp.h"
 #include "utils/strutils.hpp"
 #include "utils/time.hpp"
+#include "utils/xml4nlp_helper.h"
 #include "console/dispatcher.h"
 #include "boost/program_options.hpp"
 
@@ -21,6 +22,7 @@ using boost::program_options::parse_command_line;
 using ltp::strutils::trim;
 
 std::string type;
+std::string format;
 
 void multithreaded_ltp( void * args) {
   std::string sentence;
@@ -35,9 +37,9 @@ void multithreaded_ltp( void * args) {
     XML4NLP xml4nlp;
     xml4nlp.CreateDOMFromString(sentence);
 
-    if (type == "sp") {
+    /*if (type == LTP_SERVICE_NAME_SPLITSENT) {
       engine->splitSentence_dummy(xml4nlp);
-    } else if(type == LTP_SERVICE_NAME_SEGMENT) {
+    } else*/ if(type == LTP_SERVICE_NAME_SEGMENT) {
       engine->wordseg(xml4nlp);
     } else if(type == LTP_SERVICE_NAME_POSTAG) {
       engine->postag(xml4nlp);
@@ -52,9 +54,13 @@ void multithreaded_ltp( void * args) {
     }
 
     std::string result;
-    xml4nlp.SaveDOM(result);
-    xml4nlp.ClearDOM();
+    if (format == LTP_SERVICE_OUTPUT_FORMAT_JSON) {
+      result = ltp::utility::xml2jsonstr(xml4nlp, type);
+    } else { //xml
+      xml4nlp.SaveDOM(result);
+    }
     dispatcher->output(ret, result);
+    xml4nlp.ClearDOM();
   }
   return;
 }
@@ -77,7 +83,10 @@ int main(int argc, char *argv[]) {
      "- " LTP_SERVICE_NAME_NER ": Named entity recognization\n"
      "- " LTP_SERVICE_NAME_DEPPARSE ": Dependency parsing\n"
      "- " LTP_SERVICE_NAME_SRL ": Semantic role labeling (equals to all)\n"
-     "- all: The whole pipeline [default]")
+     "- " LTP_SERVICE_NAME_ALL ": The whole pipeline [default]")
+    ("format", value<std::string>(), "Ouput format\n"
+     "- " LTP_SERVICE_OUTPUT_FORMAT_XML " [default]\n"
+     "- " LTP_SERVICE_OUTPUT_FORMAT_JSON)
     ("input", value<std::string>(), "The path to the input file.")
     ("segmentor-model", value<std::string>(),
      "The path to the segment model [default=ltp_data/cws.model].")
@@ -118,17 +127,27 @@ int main(int argc, char *argv[]) {
     }
   }
 
-  std::string last_stage = "all";
+  std::string last_stage = LTP_SERVICE_NAME_DEFAULT;
   if (vm.count("last-stage")) {
     last_stage = vm["last-stage"].as<std::string>();
     if (last_stage != LTP_SERVICE_NAME_SEGMENT
         && last_stage != LTP_SERVICE_NAME_POSTAG
         && last_stage != LTP_SERVICE_NAME_NER
         && last_stage != LTP_SERVICE_NAME_DEPPARSE
         && last_stage != LTP_SERVICE_NAME_SRL
-        && last_stage != "all") {
-      std::cerr << "Unknown stage name:" << last_stage << ", reset to 'all'" << std::endl;
-      last_stage = "all";
+        && last_stage != LTP_SERVICE_NAME_ALL) {
+      std::cerr << "Unknown stage name:" << last_stage << ", reset to '" LTP_SERVICE_NAME_DEFAULT "'" << std::endl;
+      last_stage = LTP_SERVICE_NAME_DEFAULT;
+    }
+  }
+
+  format = LTP_SERVICE_OUTPUT_FORMAT_DEFAULT;
+  if (vm.count("format")) {
+    format = vm["format"].as<std::string>();
+    if (format != LTP_SERVICE_OUTPUT_FORMAT_XML
+        && format != LTP_SERVICE_OUTPUT_FORMAT_JSON) {
+      std::cerr << "Unknown format:" << last_stage << ", reset to '" LTP_SERVICE_OUTPUT_FORMAT_DEFAULT "'" << std::endl;
+      format = LTP_SERVICE_OUTPUT_FORMAT_DEFAULT;
     }
   }
 

src/ltp/Ltp.cpp

@@ -58,7 +58,7 @@ bool LTP::load(const std::string& last_stage,
     target_mask = (kActiveSegmentor|kActivePostagger|kActiveNER);
   } else if (last_stage == LTP_SERVICE_NAME_DEPPARSE) {
     target_mask = (kActiveSegmentor|kActivePostagger|kActiveParser);
-  } else if ((last_stage == LTP_SERVICE_NAME_SRL) || (last_stage == "all")) {
+  } else if ((last_stage == LTP_SERVICE_NAME_SRL) || (last_stage == LTP_SERVICE_NAME_ALL)) {
     target_mask =
       (kActiveSegmentor|kActivePostagger|kActiveNER|kActiveParser|kActiveSRL);
   }
@@ -122,6 +122,8 @@ bool LTP::load(const std::string& last_stage,
     return false;
   }
 
+  INFO_LOG("Resources loading finished.");
+
   return true;
 }
 

src/ltp/Ltp.h

@@ -14,11 +14,18 @@
 #define MAX_SENTENCE_LEN 1024
 #define MAX_WORDS_NUM 256
 
-#define LTP_SERVICE_NAME_SEGMENT  "ws"
-#define LTP_SERVICE_NAME_POSTAG   "pos"
-#define LTP_SERVICE_NAME_NER      "ner"
-#define LTP_SERVICE_NAME_DEPPARSE "dp"
-#define LTP_SERVICE_NAME_SRL      "srl"
+#define LTP_SERVICE_NAME_SPLITSENT "sp"
+#define LTP_SERVICE_NAME_SEGMENT   "ws"
+#define LTP_SERVICE_NAME_POSTAG    "pos"
+#define LTP_SERVICE_NAME_NER       "ner"
+#define LTP_SERVICE_NAME_DEPPARSE  "dp"
+#define LTP_SERVICE_NAME_SRL       "srl"
+#define LTP_SERVICE_NAME_ALL       "all"
+#define LTP_SERVICE_NAME_DEFAULT   LTP_SERVICE_NAME_ALL
+
+#define LTP_SERVICE_OUTPUT_FORMAT_XML "xml"
+#define LTP_SERVICE_OUTPUT_FORMAT_JSON "json"
+#define LTP_SERVICE_OUTPUT_FORMAT_DEFAULT LTP_SERVICE_OUTPUT_FORMAT_XML
 
 enum ErrorCodes {
   kEmptyStringError = 1,  /*< The input sentence is empty */

src/parser.n/CMakeLists.txt

@@ -1,6 +1,6 @@
 include_directories (${SOURCE_DIR}/
     ${THIRDPARTY_DIR}/boost/include
-    ${THIRDPARTY_DIR}/eigen-3.2.4/
+    ${THIRDPARTY_DIR}/eigen-3.2.10/
     )
 
 set (nndepparser_VERSION "0.0.1")

src/server/CMakeLists.txt

@@ -2,7 +2,8 @@ include_directories (./
     ${SOURCE_DIR}/
     ${THIRDPARTY_DIR}/boost/include/
     ${THIRDPARTY_DIR}/maxent/
-    ${THIRDPARTY_DIR}/tinyxml/)
+    ${THIRDPARTY_DIR}/tinyxml/
+    ${THIRDPARTY_DIR}/jsoncpp/include/)
 
 set (ltp_server_SRC ltp_server.cpp mongoose.c mongoose.h)
 link_directories ( ${LIBRARY_OUTPUT_PATH} )
@@ -20,4 +21,5 @@ target_link_libraries (ltp_server
     pthread
     boost_program_options_static_lib
     boost_regex_static_lib
-    dl)
+    dl
+    jsoncpp)

src/server/ltp_server.cpp

@@ -11,6 +11,8 @@
 #include "utils/strutils.hpp"
 #include "utils/logging.hpp"
 #include "utils/codecs.hpp"
+#include "utils/xml4nlp_helper.h"
+#include "json/json.h"
 
 #define POST_LEN 1024
 #define EXECUTABLE "ltp_server"
@@ -113,9 +115,9 @@ int main(int argc, char *argv[]) {
         && last_stage != LTP_SERVICE_NAME_NER
         && last_stage != LTP_SERVICE_NAME_DEPPARSE
         && last_stage != LTP_SERVICE_NAME_SRL
-        && last_stage != "all") {
+        && last_stage != LTP_SERVICE_NAME_ALL) {
       std::cerr << "Unknown stage name:" << last_stage << ", reset to 'all'" << std::endl;
-      last_stage = "all";
+      last_stage = LTP_SERVICE_NAME_ALL;
     }
   }
 
@@ -191,6 +193,9 @@ int main(int argc, char *argv[]) {
     exit(EXIT_FAILURE);
   }
 
+  INFO_LOG("Start listening on port [%s]...", port_str.c_str());
+
+
   // getchar();
   while (exit_flag == 0) {
     sleep(100000);
@@ -250,14 +255,17 @@ static void ErrorResponse(struct mg_connection* conn,
   }
 }
 
+
 static int Service(struct mg_connection *conn) {
   char *sentence;
   char type[10];
   char xml[10];
+  char format[10];
 
   std::string str_post_data;
   std::string str_type;
   std::string str_xml;
+  std::string str_format;
 
   const struct mg_request_info *ri = mg_get_request_info(conn);
 
@@ -296,6 +304,12 @@ static int Service(struct mg_connection *conn) {
                xml,
                sizeof(xml) - 1);
 
+    mg_get_var(str_post_data.c_str(),
+               str_post_data.size(),
+               "f",
+               format,
+               sizeof(format) - 1);
+
     string strSentence = sentence;
 
     // validation check
@@ -323,6 +337,12 @@ static int Service(struct mg_connection *conn) {
       str_xml = xml;
     }
 
+    if(strlen(format) == 0) {
+      str_format = "";
+    } else {
+      str_format = format;
+    }
+
     delete []sentence;
     DEBUG_LOG("Input sentence is: %s", strSentence.c_str());
 
@@ -363,7 +383,14 @@ static int Service(struct mg_connection *conn) {
         ErrorResponse(conn, static_cast<ErrorCodes>(ret));
         return 0;
       }
-    } else { // srl or all
+    } else if (str_type == LTP_SERVICE_NAME_SRL){
+      int ret = engine->srl(xml4nlp);
+      if (0 != ret) {
+        ErrorResponse(conn, static_cast<ErrorCodes>(ret));
+        return 0;
+      }
+    } else {
+      str_type = LTP_SERVICE_NAME_ALL;
       int ret = engine->srl(xml4nlp);
       if (0 != ret) {
         ErrorResponse(conn, static_cast<ErrorCodes>(ret));
@@ -374,7 +401,12 @@ static int Service(struct mg_connection *conn) {
     TRACE_LOG("Analysis is done.");
 
     std::string strResult;
-    xml4nlp.SaveDOM(strResult);
+    if (str_format == LTP_SERVICE_OUTPUT_FORMAT_JSON) {
+      strResult = ltp::utility::xml2jsonstr(xml4nlp, str_type);
+    } else { //xml
+      xml4nlp.SaveDOM(strResult);
+    }
+
 
     strResult = "HTTP/1.1 200 OK\r\n\r\n" + strResult;
     mg_printf(conn, "%s", strResult.c_str());

src/unittest/CMakeLists.txt

@@ -6,7 +6,7 @@ include_directories (./
     ${SOURCE_DIR}/
     ${THIRDPARTY_DIR}/gtest/include
     ${THIRDPARTY_DIR}/boost/include
-    ${THIRDPARTY_DIR}/eigen-3.2.4/)
+    ${THIRDPARTY_DIR}/eigen-3.2.10/)
 link_directories (${LIBRARY_OUTPUT_PATH})
 
 # add unittest executable