thread remake

.gitignore

@@ -100,6 +100,7 @@ ENV/
 # large file
 *.pkl
 *.conv
+/raw_data/
 
 # trained model
 model/

src/data_utils.py

@@ -6,7 +6,7 @@
 import random
 import numpy as np
 from tensorflow.python.client import device_lib
-from src.word_sequence import WordSequence
+from word_sequence import WordSequence
 
 # 处理词向量临界值
 VOCAB_SIZE_THRESHOLD_GPU = 50000
@@ -216,15 +216,15 @@ def batch_flow_bucket(data, ws, batch_size, raw=False, add_end=True, n_bucket=5,
 
 
 def test_batch_flow():
-    from src.fake_data import generate
+    from fake_data import generate
     x_data, y_data, ws_input, ws_target = generate(size=10000)
     flow = batch_flow([x_data, y_data], [ws_input, ws_target], 4)
     x, xl, y, yl = next(flow)
     print(x.shape, y.shape, xl.shape, yl.shape)
 
 
 def test_batch_flow_bucket():
-    from src.fake_data import generate
+    from fake_data import generate
     x_data, y_data, ws_input, ws_target = generate(size=10000)
     flow = batch_flow_bucket([x_data, y_data], [ws_input, ws_target], batch_size=4, debug=True)
     for _ in range(10):

src/extract_conv.py

@@ -8,7 +8,7 @@
 from tqdm import tqdm
 
 
-# 去掉非法字符
+# 去掉非法字符，合并句子
 def make_split(line):
     if re.match(r'.*([，…?!\.,!？])$', ''.join(line)):
         return []
@@ -35,11 +35,11 @@ def regular(sen):
 
 
 def main(limit=20, x_limit=3, y_limit=6):
-    from src.word_sequence import WordSequence
+    from word_sequence import WordSequence
 
     # 解压文件
     print('extract lines')
-    fp = open("dgk_shooter_min.conv", 'r', errors='ignore', encoding='utf-8')
+    fp = open("raw_data/dgk_shooter_min.conv", 'r', errors='ignore', encoding='utf-8')
     # 保存全部句子列表
     groups = []
     # 保存一行
@@ -59,7 +59,7 @@ def main(limit=20, x_limit=3, y_limit=6):
             line = line[:-1]
 
             group.append(list(regular(''.join(line))))
-        # E开头句子
+        # E开头句子---line.startswith('E ')
         else:
             if group:
                 groups.append(group)

src/fake_data.py

@@ -5,7 +5,7 @@
 """
 import random
 import numpy as np
-from src.word_sequence import WordSequence
+from word_sequence import WordSequence
 
 
 def generate(max_len=10, size=1000, same_len=False, seed=0):

src/seq_to_seq.py

@@ -20,8 +20,8 @@
 from tensorflow.contrib.rnn import ResidualWrapper
 # from tensorflow.contrib.rnn import LSTMStateTuple
 
-from src.word_sequence import WordSequence
-from src.data_utils import _get_embed_device
+from word_sequence import WordSequence
+from data_utils import _get_embed_device
 
 
 class SequenceToSequence(object):

src/test.py

@@ -11,8 +11,8 @@
 
 
 def test(params):
-    from src.seq_to_seq import SequenceToSequence
-    from src.data_utils import batch_flow
+    from seq_to_seq import SequenceToSequence
+    from data_utils import batch_flow
 
     x_data, _ = pickle.load(open('chatbot.pkl', 'rb'))
     ws = pickle.load(open('ws.pkl', 'rb'))

src/thread_generator.py

@@ -1,6 +1,12 @@
+# -*- coding: utf-8 -*-
+"""
+   Description : 线程生成器
+   Author :        xxm
+"""
 from threading import Thread
 from queue import Queue
 
+
 class ThreadedGenerator(object):
 
     def __init__(self, iterator,
@@ -56,8 +62,8 @@ def __next__(self):
             raise StopIteration()
         return value
 
-def test():
 
+def test():
     def gene():
         i = 0
         while True:
@@ -72,6 +78,6 @@ def gene():
 
     test.close()
 
+
 if __name__ == '__main__':
     test()
-

src/train_anti.py

@@ -10,11 +10,11 @@
 from tqdm import tqdm
 
 
-def test(params):
-    from src.seq_to_seq import SequenceToSequence
-    from src.data_utils import batch_flow_bucket as batch_flow
-    from src.word_sequence import WordSequence
-    from src.thread_generator import ThreadedGenerator
+def train(params):
+    from seq_to_seq import SequenceToSequence
+    from data_utils import batch_flow_bucket as batch_flow
+    from word_sequence import WordSequence
+    from thread_generator import ThreadedGenerator
 
     # 加载数据
     x_data, y_data = pickle.load(open('chatbot.pkl', 'rb'))
@@ -89,7 +89,6 @@ def test(params):
                 model.save(sess, save_path)
             flow.close()
 
-    # 训练2
     tf.reset_default_graph()
     model_pred = SequenceToSequence(
         input_vocab_size=len(ws),
@@ -121,6 +120,7 @@ def test(params):
             if t >= 3:
                 break
 
+    # 训练2
     tf.reset_default_graph()
     model_pred = SequenceToSequence(
         input_vocab_size=len(ws),
@@ -155,7 +155,7 @@ def test(params):
 
 def main():
     import json
-    test(json.load(open('params.json')))
+    train(json.load(open('params.json')))
 
 
 if __name__ == '__main__':