add multi-threaded test program

.gitignore

@@ -29,8 +29,7 @@ lgsrl
 otcws
 otpos
 otner
-ltp-model
-auto-test.sh
+maxent
 
 ###############
 # data file   #

examples/Makefile

@@ -1,11 +1,17 @@
-all: cws pos par ner multi_cws
+all: cws cws_cmdline pos par ner multi_cws_cmdline
 
 cws: cws.cpp
 	g++ -o cws cws.cpp -I./ \
 		-I../include/ \
 		-I../thirdparty/boost/include \
 		-L../lib/ -lsegmentor -lboost_regex
 
+cws_cmdline: cws_cmdline.cpp
+	g++ -o cws_cmdline cws_cmdline.cpp -I./ \
+		-I../include/ \
+		-I../thirdparty/boost/include \
+		-Wl,-dn -L../lib/ -lsegmentor -lboost_regex -Wl,-dy
+
 pos: pos.cpp
 	g++ -o pos pos.cpp -I./ \
 		-I../include/ \
@@ -21,20 +27,21 @@ par: par.cpp
 		-I../src/parser/ \
 		-L../lib -lparser
 
-multi_cws: multi_cws.cpp
-	g++ -o multi_cws multi_cws.cpp \
+multi_cws_cmdline: multi_cws_cmdline.cpp
+	g++ -o multi_cws_cmdline multi_cws_cmdline.cpp \
 		thirdparty/tinythreadpp/tinythread.cpp \
 		-I./ \
 		-I../include/ \
 		-I../thirdparty/boost/include/ \
 		-I./thirdparty/tinythreadpp/ \
-		-L../lib/ -lsegmentor -lboost_regex -lpthread
+		-Wl,-dn -L../lib/ -lsegmentor -lboost_regex -Wl,-dy -lpthread
 
 .PHONY: clean
 
 clean:
 	rm cws
+	rm cws_cmdline
 	rm pos
 	rm ner
 	rm par
-	rm multi_cws
+	rm multi_cws_cmdline

examples/cws_cmdline.cpp

@@ -0,0 +1,70 @@
+/*
+ * Single-threaded segmentor test program. The user input a line
+ * of Chinese sentence an the program will output its segment
+ * result.
+ *
+ *  @dependency package: tinythread - a portable c++ wrapper for
+ *                       multi-thread library.
+ *  @author:             LIU, Yijia
+ *  @data:               2013-09-24
+ *
+ * This program is special designed for UNIX user, for get time
+ * is not compilable under MSVC
+ */
+#include <iostream>
+#include <ctime>
+#include <string>
+#include <sys/time.h>
+#include <sys/types.h>
+#include "segment_dll.h"
+
+double get_time(void) {
+    struct timeval tv;
+    gettimeofday(&tv, NULL);
+    return tv.tv_sec + (tv.tv_usec / 1000000.0);
+}
+
+int main(int argc, char * argv[]) {
+    if (argc < 2) {
+        std::cerr << "cws [model path] [lexicon_file]" << std::endl;
+        return 1;
+    }
+
+    void * engine = 0;
+    if (argc == 2) {
+        engine = segmentor_create_segmentor(argv[1]);
+    } else if (argc == 3) {
+        engine = segmentor_create_segmentor(argv[1], argv[2]);
+    }
+
+    if (!engine) {
+        return -1;
+    }
+    std::vector<std::string> words;
+    std::string sentence;
+
+    std::cerr << "TRACE: Model is loaded" << std::endl;
+    double tm = get_time();
+
+    while (std::getline(std::cin, sentence, '\n')) {
+        words.clear();
+        if (sentence.size() == 0) { continue; }
+        int len = segmentor_segment(engine, sentence, words);
+        for (int i = 0; i < len; ++ i) {
+            std::cout << words[i];
+            if (i+1 == len) std::cout <<std::endl;
+            else std::cout<< "|";
+        }
+    }
+
+    segmentor_release_segmentor(engine);
+
+    tm = get_time() - tm;
+    std::cerr << "TRACE: consume "
+        << tm 
+        << " seconds."
+        << std::endl;
+
+    return 0;
+}
+

examples/multi_cws_cmdline.cpp

@@ -0,0 +1,141 @@
+/*
+ * Multi-threaded segmentor test program. The user input a line
+ * of Chinese sentence an the program will output its segment
+ * result.
+ *
+ *  @dependency package: tinythread - a portable c++ wrapper for
+ *                       multi-thread library.
+ *  @author:             LIU, Yijia
+ *  @data:               2013-09-24
+ *
+ * This program is special designed for UNIX user, for get time
+ * is not compilable under MSVC
+ */
+#include <iostream>
+#include <ctime>
+#include <vector>
+#include <list>
+#include <sys/time.h>
+#include <sys/types.h>
+
+#include "segment_dll.h"
+#include "tinythread.h"
+#include "fast_mutex.h"
+
+using namespace std;
+using namespace tthread;
+
+const int MAX_LEN = 1024;
+
+double get_time(void) {
+    struct timeval tv;
+    gettimeofday(&tv, NULL);
+    return tv.tv_sec + (tv.tv_usec / 1000000.0);
+}
+
+class Dispatcher {
+public:
+    Dispatcher( void * model ) {
+        _model = model;
+    }
+
+    int next(string &sentence) {
+        sentence = "";
+        lock_guard<fast_mutex> guard(_mutex);
+        if (!getline(cin, sentence, '\n')) {
+            return -1;
+        }
+        return 0;
+    }
+
+    void output(const vector<string> &result) {
+        lock_guard<fast_mutex> guard(_mutex);
+        for (int i = 0; i < result.size(); ++ i) {
+            cout << result[i];
+            cout << (i == result.size() - 1 ? '\n' : '|');
+        }
+        return;
+    }
+
+    void * model() {
+        return _model;
+    }
+
+private:
+    fast_mutex      _mutex;
+    void *          _model;
+    string          _sentence;
+};
+
+void multithreaded_segment( void * args) {
+    string sentence;
+    vector<string> result;
+
+    Dispatcher * dispatcher = (Dispatcher *)args;
+    void * model = dispatcher->model();
+
+    while (true) {
+        int ret = dispatcher->next(sentence);
+
+        if (ret < 0)
+            break;
+
+        result.clear();
+        segmentor_segment(model, sentence, result);
+        dispatcher->output(result);
+    }
+
+    return;
+}
+
+int main(int argc, char ** argv) {
+    if (argc < 2) {
+        std::cerr << "multi-cws [model path] [lexicon file]" << std::endl;
+    }
+
+    string sentence;
+    vector<string> result;
+
+    void * engine = 0;
+    if (argc == 2) {
+        engine = segmentor_create_segmentor(argv[1]);
+    } else if (argc == 3) {
+        engine = segmentor_create_segmentor(argv[1], argv[2]);
+    }
+
+    if (!engine) {
+        return -1;
+    }
+
+    std::vector<std::string> words;
+
+    int num_threads = thread::hardware_concurrency();
+    std::cerr << "TRACE: Model is loaded" << std::endl;
+    std::cerr << "TRACE: Running " << num_threads << " thread(s)" << std::endl;
+
+    Dispatcher * dispatcher = new Dispatcher( engine );
+
+    double tm = get_time();
+    list<thread *> thread_list;
+    for (int i = 0; i < num_threads; ++ i) {
+        thread * t = new thread( multithreaded_segment, (void *)dispatcher );
+        thread_list.push_back( t );
+    }
+
+    for (list<thread *>::iterator i = thread_list.begin();
+            i != thread_list.end(); ++ i) {
+        thread * t = *i;
+        t->join();
+        delete t;
+    }
+
+    tm = get_time() - tm;
+    std::cerr << "TRACE: consume "
+        << tm 
+        << " seconds."
+        << std::endl;
+
+
+    return 0;
+}
+