neurlux example

dtrizna · Jan 19, 2024 · d167b34 · d167b34
1 parent d858d80
commit d167b34
Show file tree

Hide file tree

Showing 5 changed files with 10,054 additions and 4 deletions.
diff --git a/nebula/models/neurlux/preprocessor.py b/nebula/models/neurlux/preprocessor.py
@@ -77,10 +77,13 @@ def encode(self, tokenized):
         """Encodes tokenized text."""
         return [self.vocab[token] for token in tokenized]
 
-    def preprocess(self, text):
+    def preprocess(self, text, pad=True, tokenize=True):
         """Preprocesses text for NeurLux."""
-        tokenized = self.tokenize(text)
-        encoded = self.encode(tokenized)
+        if tokenize:
+            text = self.tokenize(text)
+        encoded = self.encode(text)
+        if pad:
+            encoded = self.pad(encoded)
         return encoded
 
     def preprocess_sequence(self, sequence):

diff --git a/nebula/objects/neurlux_whitespace.model b/nebula/objects/neurlux_whitespace.model