Force using CPU for serving

deepqa2/dataset/textdata.py

@@ -24,6 +24,7 @@
 import os  # Checking file existance
 import random
 import json
+from utils import segmenter
 from tqdm import tqdm  # Progress bar
 from dataset.cornelldata import CornellData
 from dataset.egretdata import EgretData
@@ -334,6 +335,11 @@ def extractText(self, line, isTarget=False):
         Return:
             list<int>: the list of the word ids of the sentence
         """
+        # try:
+        #     line = segmenter.process_sentence(line).decode('utf-8')
+        #     print('Fix Line:', line)
+        # except:
+        #     pass
         words = []
 
         # Extract sentences

deepqa2/serve/api/chatbotmanager.py

@@ -28,6 +28,10 @@
 from munch import munchify
 from models.rnn import Model
 
+# force using CPU for serving
+# http://stackoverflow.com/questions/36584907/tensor-flow-toggle-between-cpu-gpu
+os.environ['CUDA_VISIBLE_DEVICES'] = ''
+
 config = Config()
 logger = logging.getLogger(__name__)
 tf.logging.set_verbosity(tf.logging.DEBUG)

deepqa2/utils/segmenter.py

@@ -0,0 +1,29 @@
+# coding:utf8
+'''  
+Segmenter with Chinese  
+'''
+
+import jieba  
+import langid
+
+
+def segment_chinese_sentence(sentence):  
+    '''
+    Return segmented sentence.
+    '''
+    seg_list = jieba.cut(sentence, cut_all=False)
+    seg_sentence = u" ".join(seg_list)
+    return seg_sentence.strip().encode('utf8')
+
+
+def process_sentence(sentence):  
+    '''
+    Only process Chinese Sentence.
+    '''
+    if langid.classify(sentence)[0] == 'zh':
+        return segment_chinese_sentence(sentence)
+    return sentence
+
+if __name__ == "__main__":  
+    print(process_sentence('飞雪连天射白鹿'))
+    print(process_sentence('I have a pen.'))