Handle wrong lines in the dictionary (empty line or unknown symbols (f…

…acebookresearch#126)
chenyahao4179 · Apr 23, 2019 · 3159355 · 3159355
1 parent 3dbfedd
commit 3159355
Showing 1 changed file with 6 additions and 2 deletions.
diff --git a/src/evaluation/word_translation.py b/src/evaluation/word_translation.py
@@ -54,9 +54,13 @@ def load_dictionary(path, word2id1, word2id2):
     not_found2 = 0
 
     with io.open(path, 'r', encoding='utf-8') as f:
-        for _, line in enumerate(f):
+        for index, line in enumerate(f):
             assert line == line.lower()
-            word1, word2 = line.rstrip().split()
+            parts = line.rstrip().split()
+            if len(parts) < 2:
+                logger.warning("Could not parse line %s (%i)", line, index)
+                continue
+            word1, word2 = parts
             if word1 in word2id1 and word2 in word2id2:
                 pairs.append((word1, word2))
             else: