Merge pull request google-research#266 from msramalho/patch-1

Tokenization code simplification
BBQLYNDP · Dec 18, 2018 · b8ba348 · b8ba348
2 parents c74caa3 + 284945c
commit b8ba348
Showing 1 changed file with 1 addition and 4 deletions.
diff --git a/tokenization.py b/tokenization.py
@@ -84,10 +84,7 @@ def load_vocab(vocab_file):
 
 def convert_by_vocab(vocab, items):
   """Converts a sequence of [tokens|ids] using the vocab."""
-  output = []
-  for item in items:
-    output.append(vocab[item])
-  return output
+  return [vocab[item] for item in items]
 
 
 def convert_tokens_to_ids(vocab, tokens):