add length check

0xuye0 · Sep 9, 2013 · 141746a · 141746a
1 parent 946512e
commit 141746a
Show file tree

Hide file tree

Showing 3 changed files with 40 additions and 20 deletions.
diff --git a/src/__ltp_dll/Ltp.cpp b/src/__ltp_dll/Ltp.cpp
@@ -21,6 +21,7 @@
 #pragma comment(lib, "srl.lib")
 #endif
 
+#include "codecs.hpp"
 #include "logging.hpp"
 #include "cfgparser.hpp"
 
@@ -206,6 +207,11 @@ int LTP::wordseg() {
         string strStn = m_xml4nlp.GetSentence(i);
         vector<string> vctWords;
 
+        if (ltp::strutils::codecs::length(strStn) > MAX_SENTENCE_LEN) {
+            ERROR_LOG("in LTP::wordseg, input sentence is too long");
+            return -1;
+        }
+
         if (0 == segmentor_segment(segmentor, strStn, vctWords)) {
             ERROR_LOG("in LTP::wordseg, failed to perform word segment on \"%s\"",
                     strStn.c_str());
@@ -257,6 +263,17 @@ int LTP::postag() {
         vector<string> vecPOS;
 
         m_xml4nlp.GetWordsFromSentence(vecWord, i);
+
+        if (0 == vecWord.size()) {
+            ERROR_LOG("Input sentence is empty.");
+            return -1;
+        }
+
+        if (vecWord.size() > MAX_WORDS_NUM) {
+            ERROR_LOG("Input sentence is too long.");
+            return -1;
+        }
+
         if (0 == postagger_postag(postagger, vecWord, vecPOS)) {
             ERROR_LOG("in LTP::postag, failed to perform postag on sent. #%d", i+1);
             return -1;
@@ -319,6 +336,16 @@ int LTP::ner() {
             return -1;
         }
 
+        if (0 == vecWord.size()) {
+            ERROR_LOG("Input sentence is empty.");
+            return -1;
+        }
+
+        if (vecWord.size() > MAX_WORDS_NUM) {
+            ERROR_LOG("Input sentence is too long.");
+            return -1;
+        }
+
         if (0 == ner_recognize(ner, vecWord, vecPOS, vecNETag)) {
             ERROR_LOG("in LTP::ner, failed to perform ner on sent. #%d", i+1);
             return -1;
@@ -373,6 +400,16 @@ int LTP::parser() {
             return -1;
         }
 
+        if (0 == vecWord.size()) {
+            ERROR_LOG("Input sentence is empty.");
+            return -1;
+        }
+
+        if (vecWord.size() > MAX_WORDS_NUM) {
+            ERROR_LOG("Input sentence is too long.");
+            return -1;
+        }
+
         if (-1 == parser_parse(parser, vecWord, vecPOS, vecHead, vecRel)) {
             ERROR_LOG("in LTP::parser, failed to perform parse on sent. #%d", i+1);
             return -1;

diff --git a/src/__ltp_dll/Ltp.h b/src/__ltp_dll/Ltp.h
@@ -15,6 +15,8 @@
 using namespace std;
 
 // extern ofstream ltp_log_file;
+#define MAX_SENTENCE_LEN 300
+#define MAX_WORDS_NUM    70
 
 class LTP {
 public:

diff --git a/src/server/ltp_server.cpp b/src/server/ltp_server.cpp
@@ -136,27 +136,8 @@ static int Service(struct mg_connection *conn) {
                 return 0;
             }
 
-            // guanrante the input sentence is not to long.
-            int  num_para = xml4nlp.CountParagraphInDocument();
-            bool flag = true;
-            for (int i = 0; i < num_para; ++ i) {
-                int num_sent = xml4nlp.CountSentenceInParagraph(i);
-                for (int j = 0; j < num_sent; ++ j) { 
-                    if (length(std::string(xml4nlp.GetSentence(i,j))) > 70) {
-                        flag = false;
-                    }
-                }
-            }
-
-            if (!flag) {
-                WARNING_LOG("Input sentence is too long");
-                return 0;
-            }
+            // move sentence validation check into each module
         } else {
-            if (length(strSentence) > 70) {
-                WARNING_LOG("Input sentence is too long");
-                return 0;
-            }
             xml4nlp.CreateDOMFromString(strSentence);
         }