Skip to content

Commit

Permalink
[safety] can supply custom offensive language file (facebookresearch#…
Browse files Browse the repository at this point in the history
…2968)

* custom safety file

* fix

* autofomrat
  • Loading branch information
jaseweston authored Aug 12, 2020
1 parent 2345a05 commit dbae333
Showing 1 changed file with 15 additions and 7 deletions.
22 changes: 15 additions & 7 deletions parlai/utils/safety.py
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,8 @@ class OffensiveStringMatcher:
def __init__(self, datapath: str = None):
"""
Get data from external sources and build data representation.
If datapath ends in '.txt' it is assumed a custom model file is already given.
"""
import parlai.core.build_data as build_data
from parlai.core.dict import DictionaryAgent
Expand Down Expand Up @@ -115,14 +117,20 @@ def build():
# Mark the data as built.
build_data.mark_done(dpath, version)

if datapath is None:
from parlai.core.params import ParlaiParser

parser = ParlaiParser(False, False)
self.datapath = os.path.join(parser.parlai_home, 'data')
if datapath is not None and datapath.endswith('.txt'):
# Load custom file.
self.datafile = datapath
else:
self.datapath = datapath
self.datafile = _path()
# Build data from zoo, and place in given datapath.
if datapath is None:
# Build data from zoo.
from parlai.core.params import ParlaiParser

parser = ParlaiParser(False, False)
self.datapath = os.path.join(parser.parlai_home, 'data')
else:
self.datapath = datapath
self.datafile = _path()

# store a token trie: e.g.
# {'2': {'girls': {'1': {'cup': {'__END__': True}}}}
Expand Down

0 comments on commit dbae333

Please sign in to comment.