get_vocabulary: don't crash on double whitespace or empty line

jennifer1995 · Mar 26, 2018 · aa1bd9f · aa1bd9f
1 parent 7bb1c3d
commit aa1bd9f
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 2 deletions.
diff --git a/get_vocab.py b/get_vocab.py
@@ -7,7 +7,8 @@
 
 for line in sys.stdin:
     for word in line.strip().split(' '):
-        c[word] += 1
+        if word:
+            c[word] += 1
 
 for key,f in sorted(c.items(), key=lambda x: x[1], reverse=True):
     print(key+" "+ str(f))
diff --git a/learn_bpe.py b/learn_bpe.py
@@ -66,7 +66,8 @@ def get_vocabulary(fobj, is_dict=False):
             vocab[word] += int(count)
         else:
             for word in line.strip().split(' '):
-                vocab[word] += 1
+                if word:
+                    vocab[word] += 1
     return vocab
 
 def update_pair_statistics(pair, changed, stats, indices):