clean the code

emb_build/tran_emb.py

@@ -0,0 +1,27 @@
+import word2vec
+import fire
+
+paths = ['raw_word.txt', 'raw_article.txt']
+sizes = [300]
+
+
+def tran(path):
+    model = word2vec.load(path)
+    vocab, vectors = model.vocab, model.vectors
+    print(path)
+    print('shape of word embeddings : ')
+    print(vectors.shape)
+
+    new_path = path.split('.')[0] + '_.txt'
+    print('Transform start....')
+    f = open(new_path, 'w')
+    for word, vector in zip(vocab, vectors):
+        f.write(str(word) + ' ' + ' '.join(map(str, vector)) + '\n')
+    print('Transform Complete!\n')
+
+
+for path in paths:
+    for size in sizes:
+        emb_path = path.split('.')[0].split('_')[1] + '_' + str(size) + '.bin'
+        word2vec.word2vec(path, emb_path, min_count=5, size=size, verbose=True)
+        tran(emb_path)

fine_tune.py

@@ -10,7 +10,7 @@
 import time
 import torch.nn.functional as F
 import models
-import util
+import data
 import pandas as pd
 import os
 import fire

gen_result.py

@@ -9,7 +9,7 @@
 import torch
 import models
 from config import DefaultConfig
-import util
+import data
 import fire
 import numpy as np
 import torch.nn.functional as F

main.py

@@ -10,7 +10,7 @@
 import time
 import torch.nn.functional as F
 import models
-import util
+import data
 from config import DefaultConfig
 import pandas as pd
 import os
@@ -30,7 +30,7 @@ def main(**kwargs):
         args.device = None
         torch.manual_seed(args.seed)  # set random seed for cpu
 
-    train_iter, val_iter, test_iter, args.vocab_size, vectors = util.load_data(args)
+    train_iter, val_iter, test_iter, args.vocab_size, vectors = data.load_data(args)
 
     args.print_config()
 

main_boost.py

@@ -10,7 +10,7 @@
 import time
 import torch.nn.functional as F
 import models
-import util
+import data
 from config import DefaultConfig
 import pandas as pd
 import os

test_ensemble.py

@@ -8,7 +8,7 @@
 
 import torch
 import models
-import util
+import data
 import numpy as np
 import torch.nn.functional as F
 import pandas as pd

util/split_val.py

@@ -0,0 +1,18 @@
+import pandas as pd
+import numpy as np
+import random
+
+train_data = pd.read_csv('/data/yujun/captain/datasets1/train_set.csv')
+val_data = pd.read_csv('/data/yujun/captain/datasets1/val_set.csv')
+test_data = pd.read_csv('/data/yujun/datasets/daguanbei_data/test_set.csv')
+print('raw data loaded!')
+
+train_data[['word_seg', 'class']].to_csv('word/train_set.csv')
+val_data[['word_seg', 'class']].to_csv('word/val_set.csv')
+test_data[['id', 'word_seg']].to_csv('word/test_set.csv')
+print('word data made!')
+
+train_data[['article', 'class']].to_csv('article/train_set.csv')
+val_data[['article', 'class']].to_csv('article/val_set.csv')
+test_data[['id', 'article']].to_csv('article/test_set.csv')
+print('article data made!')