Merge pull request HIT-SCIR#26 from niuox/master

multi-threads in server and __ltp_dll
0xuye0 · Sep 26, 2013 · 546b425 · 546b425
2 parents 9b5441f + f039e56
commit 546b425
Show file tree

Hide file tree

Showing 9 changed files with 243 additions and 81 deletions.
diff --git a/src/__ltp_dll/Ltp.cpp b/src/__ltp_dll/Ltp.cpp
@@ -34,32 +34,30 @@ const unsigned int LTP::DO_PARSER = 1 << 4;
 const unsigned int LTP::DO_SRL = 1 << 6;
 
 // create a platform
-LTP::LTP(XML4NLP &xml4nlp) :
+LTP::LTP() :
     m_ltpResource(),
-    m_ltpOption(),
-    m_xml4nlp(xml4nlp) {
+    m_ltpOption(){
     ReadConfFile();
 }
 
-LTP::LTP(const char * config, XML4NLP & xml4nlp) :
+LTP::LTP(const char * config) :
     m_ltpResource(),
-    m_ltpOption(),
-    m_xml4nlp(xml4nlp) {
+    m_ltpOption(){
     ReadConfFile(config);
 }
 
 LTP::~LTP() {
 }
 
-int LTP::CreateDOMFromTxt(const char * cszTxtFileName) {
+int LTP::CreateDOMFromTxt(const char * cszTxtFileName,XML4NLP &   m_xml4nlp) {
     return m_xml4nlp.CreateDOMFromFile(cszTxtFileName);
 }
 
-int LTP::CreateDOMFromXml(const char * cszXmlFileName) {
+int LTP::CreateDOMFromXml(const char * cszXmlFileName,XML4NLP &   m_xml4nlp) {
     return m_xml4nlp.LoadXMLFromFile(cszXmlFileName);
 }
 
-int LTP::SaveDOM(const char * cszSaveFileName) {
+int LTP::SaveDOM(const char * cszSaveFileName,XML4NLP &   m_xml4nlp) {
     return m_xml4nlp.SaveDOM(cszSaveFileName);
 }
 
@@ -131,13 +129,39 @@ int LTP::ReadConfFile(const char * config_file) {
         m_ltpOption.neOpt.isNum = 1;
     else
         m_ltpOption.neOpt.isNum = atoi( it->second.c_str() );*/
-
+
+    //load segmentor model
+    if (0 != m_ltpResource.LoadSegmentorResource(m_ltpOption.segmentor_model_path)) {
+        ERROR_LOG("in LTP::wordseg, failed to load segmentor resource");
+        return -1;
+    }
+    //load postagger model
+    if (0 != m_ltpResource.LoadPostaggerResource(m_ltpOption.postagger_model_path)) {
+        ERROR_LOG("in LTP::postag, failed to load postagger resource.");
+        return -1;
+    }
+    //load ner model
+    if (0 != m_ltpResource.LoadNEResource(m_ltpOption.ner_model_path)) {
+        ERROR_LOG("in LTP::ner, failed to load ner resource");
+        return -1;
+    }
+    //load paser model
+    if ( 0 != m_ltpResource.LoadParserResource(m_ltpOption.parser_model_path) ) {
+        ERROR_LOG("in LTP::parser, failed to load parser resource");
+        return -1;
+    }
+    //load srl model
+    if ( 0 != m_ltpResource.LoadSRLResource(m_ltpOption.srl_data_dir) ) {
+        ERROR_LOG("in LTP::srl, failed to load srl resource");
+        return -1;
+    }
+
     return 0;
 }
 
 // If you do NOT split sentence explicitly,
 // this will be called according to dependencies among modules
-int LTP::splitSentence_dummy() {
+int LTP::splitSentence_dummy(XML4NLP &   m_xml4nlp) {
     if ( m_xml4nlp.QueryNote(NOTE_SENT) ) {
         return 0;
     }
@@ -173,13 +197,13 @@ int LTP::splitSentence_dummy() {
 }
 
 // integrate word segmentor into LTP
-int LTP::wordseg() {
+int LTP::wordseg(XML4NLP &   m_xml4nlp) {
     if (m_xml4nlp.QueryNote(NOTE_WORD)) {
         return 0;
     }
 
     //
-    if (0 != splitSentence_dummy()) {
+    if (0 != splitSentence_dummy(m_xml4nlp)) {
         ERROR_LOG("in LTP::wordseg, failed to perform split sentence preprocess.");
         return -1;
     }
@@ -229,13 +253,13 @@ int LTP::wordseg() {
 }
 
 // integrate postagger into LTP
-int LTP::postag() {
+int LTP::postag(XML4NLP &   m_xml4nlp) {
     if ( m_xml4nlp.QueryNote(NOTE_POS) ) {
         return 0;
     }
 
     // dependency
-    if (0 != wordseg()) {
+    if (0 != wordseg(m_xml4nlp)) {
         ERROR_LOG("in LTP::postag, failed to perform word segment preprocess");
         return -1;
     }
@@ -291,13 +315,13 @@ int LTP::postag() {
 }
 
 // perform ner over xml
-int LTP::ner() {
+int LTP::ner(XML4NLP &   m_xml4nlp) {
     if ( m_xml4nlp.QueryNote(NOTE_NE) ) {
         return 0;
     }
 
     // dependency
-    if (0 != postag()) {
+    if (0 != postag(m_xml4nlp)) {
         ERROR_LOG("in LTP::ner, failed to perform postag preprocess");
         return -1;
     }
@@ -358,10 +382,10 @@ int LTP::ner() {
     return 0;
 }
 
-int LTP::parser() {
+int LTP::parser(XML4NLP &   m_xml4nlp) {
     if ( m_xml4nlp.QueryNote(NOTE_PARSER) ) return 0;
 
-    if (0 != postag()) {
+    if (0 != postag(m_xml4nlp)) {
         ERROR_LOG("in LTP::parser, failed to perform postag preprocessing");
         return -1;
     }
@@ -426,16 +450,16 @@ int LTP::parser() {
     return 0;
 }
 
-int LTP::srl() {
+int LTP::srl(XML4NLP &   m_xml4nlp) {
     if ( m_xml4nlp.QueryNote(NOTE_SRL) ) return 0;
 
     // dependency
-    if (0 != ner()) {
+    if (0 != ner(m_xml4nlp)) {
         ERROR_LOG("in LTP::srl, failed to perform ner preprocess");
         return -1;
     }
 
-    if (0 != parser()) {
+    if (0 != parser(m_xml4nlp)) {
         ERROR_LOG("in LTP::srl, failed to perform parsing preprocess");
         return -1;
     }

diff --git a/src/__ltp_dll/Ltp.h b/src/__ltp_dll/Ltp.h
@@ -20,28 +20,27 @@ using namespace std;
 
 class LTP {
 public:
-    LTP(XML4NLP & xml4nlp);
-    LTP(const char * cfg_file, XML4NLP & xml4nlp);
+    LTP();
+    LTP(const char * cfg_file);
     ~LTP();
 
-    int CreateDOMFromTxt(const char *cszTxtFileName);
-    int CreateDOMFromXml(const char *cszXmlFileName);
-    int SaveDOM(const char *cszSaveFileName);
+    int CreateDOMFromTxt(const char *cszTxtFileName,XML4NLP &   m_xml4nlp);
+    int CreateDOMFromXml(const char *cszXmlFileName,XML4NLP &   m_xml4nlp);
+    int SaveDOM(const char *cszSaveFileName,XML4NLP &   m_xml4nlp);
 
-    int wordseg();
-    int postag();
-    int ner();
-    int parser();
-    int srl();
+    int wordseg(XML4NLP &   m_xml4nlp);
+    int postag(XML4NLP &   m_xml4nlp);
+    int ner(XML4NLP &   m_xml4nlp);
+    int parser(XML4NLP &   m_xml4nlp);
+    int srl(XML4NLP &   m_xml4nlp);
 
 private:
-    int splitSentence_dummy();
+    int splitSentence_dummy(XML4NLP &   m_xml4nlp);
     int ReadConfFile(const char *confFileName = "conf/ltp.cnf");
 
 private:
     LTPResource m_ltpResource;
     LTPOption   m_ltpOption;
-    XML4NLP &   m_xml4nlp;
 
     static const unsigned int DO_XML;
     static const unsigned int DO_SPLITSENTENCE;

diff --git a/src/segmentor/segment_dll.h b/src/segmentor/segment_dll.h
@@ -1,3 +1,6 @@
+/*
+ * this is a test.
+ */
 #ifndef __LTP_SEGMENT_DLL_H__
 #define __LTP_SEGMENT_DLL_H__
 

diff --git a/src/server/ltp_server.cpp b/src/server/ltp_server.cpp
@@ -25,8 +25,7 @@
 using namespace std;
 using namespace ltp::strutils::codecs;
 
-static XML4NLP xml4nlp;
-static LTP engine(xml4nlp);
+static LTP engine;
 
 static int exit_flag;
 
@@ -130,6 +129,9 @@ static int Service(struct mg_connection *conn) {
 
         TRACE_LOG("Input sentence is: %s", strSentence.c_str());
 
+        //Get a XML4NLP instance here.
+        XML4NLP    xml4nlp;
+
         if(str_xml == "y"){
             if (-1 == xml4nlp.LoadXMLFromString(strSentence)) {
                 // failed the xml validation check
@@ -144,17 +146,17 @@ static int Service(struct mg_connection *conn) {
         TRACE_LOG("XML Creation is done.");
 
         if(str_type == "ws"){
-            engine.wordseg();
+            engine.wordseg(xml4nlp);
         } else if(str_type == "pos"){
-            engine.postag();
+            engine.postag(xml4nlp);
         } else if(str_type == "ner"){
-            engine.ner();
+            engine.ner(xml4nlp);
         } else if(str_type == "dp"){
-            engine.parser();
+            engine.parser(xml4nlp);
         } else if(str_type == "srl"){
-            engine.srl();
+            engine.srl(xml4nlp);
         } else {
-            engine.srl();
+            engine.srl(xml4nlp);
         }
 
         TRACE_LOG("Analysis is done.");

diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
@@ -5,6 +5,7 @@ include_directories (./
     ${SOURCE_DIR}/_ner
     ${SOURCE_DIR}/_srl
     ${SOURCE_DIR}/__ltp_dll
+    ${PROJECT_SOURCE_DIR}/examples/thirdparty/tinythreadpp
     ${THIRDPARTY_DIR}/crfpp
     ${THIRDPARTY_DIR}/maxent
     ${THIRDPARTY_DIR}/tinyxml)
@@ -16,27 +17,29 @@ set (ltp_test_SRC           ltp_test.cpp)
 link_directories ( ${LIBRARY_OUTPUT_PATH} )
 
 add_executable (ltp_test     ${ltp_test_SRC})
+add_executable (multi_ltp_test    multi_ltp_test.cpp ../examples/thirdparty/tinythreadpp/tinythread.cpp)
 #add_executable (ltp_test_xml ${ltp_test_xml_SRC})
 #add_executable (ltp_test2    ${ltp_test2_SRC})
 
 set (ltp_test_LIBS 
-	maxent_static_lib
-	util_static_lib
-	ltp
-	splitsnt
-	segmentor_static_lib
-	postagger_static_lib
-	parser_static_lib
-	ner_static_lib
-	srl_static_lib
-	xml4nlp
-	boost_regex_static_lib)
-	
+maxent_static_lib
+util_static_lib
+ltp
+splitsnt
+segmentor_static_lib
+postagger_static_lib
+parser_static_lib
+ner_static_lib
+srl_static_lib
+xml4nlp
+boost_regex_static_lib)
+
 if (NOT WIN32) 
 set (ltp_test_LIBS ${ltp_test_LIBS} pthread)
 endif()
 
 target_link_libraries (ltp_test ${ltp_test_LIBS})
+target_link_libraries (multi_ltp_test ${ltp_test_LIBS} pthread)
 
 #target_link_libraries (ltp_test2
 #    maxent
@@ -73,5 +76,4 @@ target_link_libraries (ltp_test ${ltp_test_LIBS})
 #add_test (NAME ltp_test
 #    COMMAND "${CMAKE_COMMAND}"
 #    -DTEST_PROG=${EXECUTABLE_TEST_OUTPUT}
-#    -P ${CMAKE_MODULE_PATH}/LtpDiff.cmake)
-
+#    -P ${CMAKE_MODULE_PATH}/LtpDiff.cmake)
diff --git a/test/ltp_test.cpp b/test/ltp_test.cpp
@@ -18,8 +18,8 @@ int main(int argc, char *argv[]) {
     string sentence;
     // ofstream log_file("test.log");
 
-    XML4NLP xml4nlp;
-    LTP ltp(argv[1], xml4nlp);
+
+    LTP ltp(argv[1]);
 
     string type(argv[2]);
     ifstream in(argv[3]);
@@ -40,21 +40,21 @@ int main(int argc, char *argv[]) {
         sentence = sentence.substr(0, len);
 
         cout << "Input sentence is: " << sentence << endl;
-
+        XML4NLP xml4nlp;
         xml4nlp.CreateDOMFromString(sentence);
 
         if(type == "ws"){
-            ltp.wordseg();
+            ltp.wordseg(xml4nlp);
         } else if(type == "pos"){
-            ltp.postag();
+            ltp.postag(xml4nlp);
         } else if(type == "ner"){
-            ltp.ner();
+            ltp.ner(xml4nlp);
         } else if(type == "dp"){
-            ltp.parser();
+            ltp.parser(xml4nlp);
         } else if(type == "srl"){
-            ltp.srl();
+            ltp.srl(xml4nlp);
         } else {
-            ltp.srl();
+            ltp.srl(xml4nlp);
         }
 
         string result;

diff --git a/test/ltp_test2.cpp b/test/ltp_test2.cpp
@@ -10,8 +10,8 @@
 
 using namespace std;
 
-static XML4NLP xml4nlp;
-static LTP ltp(xml4nlp);
+XML4NLP xml4nlp;
+static LTP ltp;
 
 int main(int argc, char *argv[])
 {
@@ -28,17 +28,17 @@ int main(int argc, char *argv[])
     xml4nlp.CreateDOMFromFile(in_file.c_str());
 
     if (type == "ws") {
-        ltp.crfWordSeg();
+        ltp.crfWordSeg(xml4nlp);
     } else if(type == "pos"){
-        ltp.postag();
+        ltp.postag(xml4nlp);
     } else if(type == "ner"){
-        ltp.ner();
+        ltp.ner(xml4nlp);
     } else if(type == "dp"){
-        ltp.gparser();
+        ltp.gparser(xml4nlp);
     } else if(type == "srl"){
-        ltp.srl();
+        ltp.srl(xml4nlp);
     } else {
-        ltp.srl();
+        ltp.srl(xml4nlp);
     }
 
     string result;