Skip to content

Commit

Permalink
adding paths to caches to logs
Browse files Browse the repository at this point in the history
  • Loading branch information
@bmccann
bmccann committed Aug 31, 2018
1 parent a6349f4 commit a62377d
Showing 1 changed file with 28 additions and 0 deletions.
28 changes: 28 additions & 0 deletions text/torchtext/datasets/generic.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@ def __init__(self, path, field, subsample=None, **kwargs):

cache_name = os.path.join(os.path.dirname(path), '.cache', os.path.basename(path), str(subsample))
if os.path.exists(cache_name):
print(f'Loading cached data from {cache_name}')
examples = torch.load(cache_name)
else:
for label in ['pos', 'neg']:
Expand All @@ -62,6 +63,7 @@ def __init__(self, path, field, subsample=None, **kwargs):
if subsample is not None and len(examples) > subsample:
break
os.makedirs(os.path.dirname(cache_name), exist_ok=True)
print(f'Caching data to {cache_name}')
torch.save(examples, cache_name)
super(imdb.IMDb, self).__init__(examples, fields, **kwargs)

Expand Down Expand Up @@ -97,6 +99,7 @@ def __init__(self, path, field, subsample=None, **kwargs):

examples = []
if os.path.exists(cache_name):
print(f'Loading cached data from {cache_name}')
examples = torch.load(cache_name)
else:
labels = ['negative', 'positive']
Expand All @@ -115,6 +118,7 @@ def __init__(self, path, field, subsample=None, **kwargs):
break

os.makedirs(os.path.dirname(cache_name), exist_ok=True)
print(f'Caching data to {cache_name}')
torch.save(examples, cache_name)

self.examples = examples
Expand Down Expand Up @@ -155,6 +159,7 @@ def __init__(self, path, exts, field, subsample=None, **kwargs):
cache_name = os.path.join(os.path.dirname(path), '.cache', os.path.basename(path), str(subsample))

if os.path.exists(cache_name):
print(f'Loading cached data from {cache_name}')
examples = torch.load(cache_name)
else:
langs = {'.de': 'German', '.en': 'English', '.fr': 'French', '.ar': 'Arabic', '.cs': 'Czech'}
Expand All @@ -176,6 +181,7 @@ def __init__(self, path, exts, field, subsample=None, **kwargs):


os.makedirs(os.path.dirname(cache_name), exist_ok=True)
print(f'Caching data to {cache_name}')
torch.save(examples, cache_name)
super(translation.TranslationDataset, self).__init__(examples, fields, **kwargs)

Expand Down Expand Up @@ -207,6 +213,7 @@ def __init__(self, path, field, subsample=None, **kwargs):

examples, all_answers = [], []
if os.path.exists(cache_name):
print(f'Loading cached data from {cache_name}')
examples, all_answers = torch.load(cache_name)
else:
with open(os.path.expanduser(path)) as f:
Expand Down Expand Up @@ -295,6 +302,7 @@ def __init__(self, path, field, subsample=None, **kwargs):
break

os.makedirs(os.path.dirname(cache_name), exist_ok=True)
print(f'Caching data to {cache_name}')
torch.save((examples, all_answers), cache_name)


Expand Down Expand Up @@ -361,6 +369,7 @@ def __init__(self, path, field, one_answer=True, subsample=None, **kwargs):

examples = []
if os.path.exists(cache_name):
print(f'Loading cached data from {cache_name}')
examples = torch.load(cache_name)
else:
with open(os.path.expanduser(path)) as f:
Expand All @@ -374,6 +383,7 @@ def __init__(self, path, field, one_answer=True, subsample=None, **kwargs):
if subsample is not None and len(examples) >= subsample:
break
os.makedirs(os.path.dirname(cache_name), exist_ok=True)
print(f'Caching data to {cache_name}')
torch.save(examples, cache_name)

super(Summarization, self).__init__(examples, fields, **kwargs)
Expand Down Expand Up @@ -508,6 +518,7 @@ def __init__(self, path, field, one_answer=True, subsample=None, **kwargs):

cache_name = os.path.join(os.path.dirname(path), '.cache', os.path.basename(path), str(subsample))
if os.path.exists(cache_name):
print(f'Loading cached data from {cache_name}')
examples, all_answers = torch.load(cache_name)
else:

Expand Down Expand Up @@ -544,6 +555,7 @@ def __init__(self, path, field, one_answer=True, subsample=None, **kwargs):
break

os.makedirs(os.path.dirname(cache_name), exist_ok=True)
print(f'Caching data to {cache_name}')
torch.save((examples, all_answers), cache_name)

super(WikiSQL, self).__init__(examples, fields, **kwargs)
Expand Down Expand Up @@ -620,6 +632,7 @@ def __init__(self, path, field, one_answer=True, subsample=None, **kwargs):

examples, all_answers = [], []
if os.path.exists(cache_name):
print(f'Loading cached data from {cache_name}')
examples, all_answers = torch.load(cache_name)
else:
with open(os.path.expanduser(path)) as f:
Expand All @@ -636,6 +649,7 @@ def __init__(self, path, field, one_answer=True, subsample=None, **kwargs):
if subsample is not None and len(examples) >= subsample:
break
os.makedirs(os.path.dirname(cache_name), exist_ok=True)
print(f'Caching data to {cache_name}')
torch.save((examples, all_answers), cache_name)

FIELD = data.Field(batch_first=True, use_vocab=False, sequential=False,
Expand Down Expand Up @@ -770,6 +784,7 @@ def __init__(self, path, field, subsample=None, **kwargs):

cache_name = os.path.join(os.path.dirname(path), '.cache', os.path.basename(path), str(subsample))
if os.path.exists(cache_name):
print(f'Loading cached data from {cache_name}')
examples = torch.load(cache_name)
else:
examples = []
Expand All @@ -783,6 +798,7 @@ def __init__(self, path, field, subsample=None, **kwargs):
if subsample is not None and len(examples) >= subsample:
break
os.makedirs(os.path.dirname(cache_name), exist_ok=True)
print(f'Caching data to {cache_name}')
torch.save(examples, cache_name)

super(WinogradSchema, self).__init__(examples, fields, **kwargs)
Expand Down Expand Up @@ -885,6 +901,7 @@ def __init__(self, path, field, subsample=None, description='woz.en', **kwargs):
examples, all_answers = [], []
cache_name = os.path.join(os.path.dirname(path), '.cache', os.path.basename(path), str(subsample), description)
if os.path.exists(cache_name):
print(f'Loading cached data from {cache_name}')
examples, all_answers = torch.load(cache_name)
else:
with open(os.path.expanduser(path)) as f:
Expand All @@ -900,6 +917,7 @@ def __init__(self, path, field, subsample=None, description='woz.en', **kwargs):
if subsample is not None and len(examples) >= subsample:
break
os.makedirs(os.path.dirname(cache_name), exist_ok=True)
print(f'Caching data to {cache_name}')
torch.save((examples, all_answers), cache_name)

super(WOZ, self).__init__(examples, fields, **kwargs)
Expand Down Expand Up @@ -998,6 +1016,7 @@ def __init__(self, path, field, subsample=None, description='multinli.in.out', *

cache_name = os.path.join(os.path.dirname(path), '.cache', os.path.basename(path), str(subsample), description)
if os.path.exists(cache_name):
print(f'Loading cached data from {cache_name}')
examples = torch.load(cache_name)
else:
examples = []
Expand All @@ -1012,6 +1031,7 @@ def __init__(self, path, field, subsample=None, description='multinli.in.out', *
if subsample is not None and len(examples) >= subsample:
break
os.makedirs(os.path.dirname(cache_name), exist_ok=True)
print(f'Caching data to {cache_name}')
torch.save(examples, cache_name)

super(MultiNLI, self).__init__(examples, fields, **kwargs)
Expand Down Expand Up @@ -1075,6 +1095,7 @@ def __init__(self, path, field, subsample=None, **kwargs):

cache_name = os.path.join(os.path.dirname(path), '.cache', os.path.basename(path), str(subsample))
if os.path.exists(cache_name):
print(f'Loading cached data from {cache_name}')
examples = torch.load(cache_name)
else:
examples = []
Expand All @@ -1089,6 +1110,7 @@ def __init__(self, path, field, subsample=None, **kwargs):
if subsample is not None and len(examples) >= subsample:
break
os.makedirs(os.path.dirname(cache_name), exist_ok=True)
print(f'Caching data to {cache_name}')
torch.save(examples, cache_name)

super().__init__(examples, fields, **kwargs)
Expand Down Expand Up @@ -1200,6 +1222,7 @@ def __init__(self, path, field, one_answer=True, subsample=None, path_to_files='

cache_name = os.path.join(os.path.dirname(path), '.cache', os.path.basename(path), str(subsample), subtask, str(nones))
if os.path.exists(cache_name):
print(f'Loading cached data from {cache_name}')
examples = torch.load(cache_name)
else:
examples = []
Expand All @@ -1219,6 +1242,7 @@ def __init__(self, path, field, one_answer=True, subsample=None, path_to_files='
if subsample is not None and len(examples) >= subsample:
break
os.makedirs(os.path.dirname(cache_name), exist_ok=True)
print(f'Caching data to {cache_name}')
torch.save(examples, cache_name)

super(OntoNotesNER, self).__init__(examples, fields, **kwargs)
Expand Down Expand Up @@ -1383,6 +1407,7 @@ def __init__(self, path, field, subsample=None, **kwargs):

cache_name = os.path.join(os.path.dirname(path), '.cache', os.path.basename(path), str(subsample))
if os.path.exists(cache_name):
print(f'Loading cached data from {cache_name}')
examples = torch.load(cache_name)
else:
examples = []
Expand All @@ -1398,6 +1423,7 @@ def __init__(self, path, field, subsample=None, **kwargs):
if subsample is not None and len(examples) >= subsample:
break
os.makedirs(os.path.dirname(cache_name), exist_ok=True)
print(f'Caching data to {cache_name}')
torch.save(examples, cache_name)

super().__init__(examples, fields, **kwargs)
Expand Down Expand Up @@ -1448,6 +1474,7 @@ def __init__(self, path, field, subsample=None, **kwargs):

examples = []
if os.path.exists(cache_name):
print(f'Loading cached data from {cache_name}')
examples = torch.load(cache_name)
else:
with open(os.path.expanduser(path)) as f:
Expand All @@ -1461,6 +1488,7 @@ def __init__(self, path, field, subsample=None, **kwargs):
if subsample is not None and len(examples) >= subsample:
break
os.makedirs(os.path.dirname(cache_name), exist_ok=True)
print(f'Caching data to {cache_name}')
torch.save(examples, cache_name)

super(JSON, self).__init__(examples, fields, **kwargs)
Expand Down

0 comments on commit a62377d

Please sign in to comment.